<!DOCTYPE HTML>
<html lang="zh-CN">


<head>
    <meta name="baidu-site-verification" content="code-KNXLvfbWBj" />
    <meta charset="utf-8">
    <meta name="keywords" content="Hadoop搭建, Aunean&#39;s Blog">
    <meta name="description" content="">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=no">
    <meta name="renderer" content="webkit|ie-stand|ie-comp">
    <meta name="mobile-web-app-capable" content="yes">
    <meta name="format-detection" content="telephone=no">
    <meta name="apple-mobile-web-app-capable" content="yes">
    <meta name="apple-mobile-web-app-status-bar-style" content="black-translucent">
    <!-- Global site tag (gtag.js) - Google Analytics -->


    <title>Hadoop搭建 | Aunean&#39;s Blog</title>
    <link rel="icon" type="image/png" href="/favicon.png">

    <link rel="stylesheet" type="text/css" href="/libs/awesome/css/all.css">
    <link rel="stylesheet" type="text/css" href="/libs/materialize/materialize.min.css">
    <link rel="stylesheet" type="text/css" href="/libs/aos/aos.css">
    <link rel="stylesheet" type="text/css" href="/libs/animate/animate.min.css">
    <link rel="stylesheet" type="text/css" href="/libs/lightGallery/css/lightgallery.min.css">
    <link rel="stylesheet" type="text/css" href="/css/matery.css">
    <link rel="stylesheet" type="text/css" href="/css/my.css">

    <script src="/libs/jquery/jquery.min.js"></script>

<meta name="generator" content="Hexo 5.4.0">
<style>.github-emoji { position: relative; display: inline-block; width: 1.2em; min-height: 1.2em; overflow: hidden; vertical-align: top; color: transparent; }  .github-emoji > span { position: relative; z-index: 10; }  .github-emoji img, .github-emoji .fancybox { margin: 0 !important; padding: 0 !important; border: none !important; outline: none !important; text-decoration: none !important; user-select: none !important; cursor: auto !important; }  .github-emoji img { height: 1.2em !important; width: 1.2em !important; position: absolute !important; left: 50% !important; top: 50% !important; transform: translate(-50%, -50%) !important; user-select: none !important; cursor: auto !important; } .github-emoji-fallback { color: inherit; } .github-emoji-fallback img { opacity: 0 !important; }</style>
<link rel="alternate" href="/atom.xml" title="Aunean's Blog" type="application/atom+xml">
<link rel="stylesheet" href="/css/prism-tomorrow.css" type="text/css"></head>





<body>
    <header class="navbar-fixed">
    <nav id="headNav" class="bg-color nav-transparent">
        <div id="navContainer" class="nav-wrapper container">
            <div class="brand-logo">
                <a href="/" class="waves-effect waves-light">
                    
                    <img src="/medias/logo.png" class="logo-img" alt="LOGO">
                    
                    <span class="logo-span">Aunean&#39;s Blog</span>
                </a>
            </div>
            

<a href="#" data-target="mobile-nav" class="sidenav-trigger button-collapse"><i class="fas fa-bars"></i></a>
<ul class="right nav-menu">
  
  <li class="hide-on-med-and-down nav-item">
    
    <a href="/" class="waves-effect waves-light">
      
      <i class="fas fa-home" style="zoom: 0.6;"></i>
      
      <span>首页</span>
    </a>
    
  </li>
  
  <li class="hide-on-med-and-down nav-item">
    
    <a href="/tags" class="waves-effect waves-light">
      
      <i class="fas fa-tags" style="zoom: 0.6;"></i>
      
      <span>标签</span>
    </a>
    
  </li>
  
  <li class="hide-on-med-and-down nav-item">
    
    <a href="/categories" class="waves-effect waves-light">
      
      <i class="fas fa-bookmark" style="zoom: 0.6;"></i>
      
      <span>分类</span>
    </a>
    
  </li>
  
  <li class="hide-on-med-and-down nav-item">
    
    <a href="/archives" class="waves-effect waves-light">
      
      <i class="fas fa-archive" style="zoom: 0.6;"></i>
      
      <span>归档</span>
    </a>
    
  </li>
  
  <li class="hide-on-med-and-down nav-item">
    
    <a href="/about" class="waves-effect waves-light">
      
      <i class="fas fa-user-circle" style="zoom: 0.6;"></i>
      
      <span>关于</span>
    </a>
    
  </li>
  
  <li class="hide-on-med-and-down nav-item">
    
    <a href="/contact" class="waves-effect waves-light">
      
      <i class="fas fa-comments" style="zoom: 0.6;"></i>
      
      <span>留言板</span>
    </a>
    
  </li>
  
  <li class="hide-on-med-and-down nav-item">
    
    <a href="/friends" class="waves-effect waves-light">
      
      <i class="fas fa-address-book" style="zoom: 0.6;"></i>
      
      <span>友情链接</span>
    </a>
    
  </li>
  
  <li class="hide-on-med-and-down nav-item">
    
    <a href="/navigate" class="waves-effect waves-light">
      
      <i class="fas fa-location-arrow" style="zoom: 0.6;"></i>
      
      <span>导航</span>
    </a>
    
  </li>
  
  <li>
    <a href="#searchModal" class="modal-trigger waves-effect waves-light">
      <i id="searchIcon" class="fas fa-search" title="搜索" style="zoom: 0.85;"></i>
    </a>
  </li>
</ul>


<div id="mobile-nav" class="side-nav sidenav">

    <div class="mobile-head bg-color">
        
        <img src="/medias/logo.png" class="logo-img circle responsive-img">
        
        <div class="logo-name">Aunean&#39;s Blog</div>
        <div class="logo-desc">
            
            Never really desperate, only the lost of the soul.
            
        </div>
    </div>

    

    <ul class="menu-list mobile-menu-list">
        
        <li class="m-nav-item">
	  
		<a href="/" class="waves-effect waves-light">
			
			    <i class="fa-fw fas fa-home"></i>
			
			首页
		</a>
          
        </li>
        
        <li class="m-nav-item">
	  
		<a href="/tags" class="waves-effect waves-light">
			
			    <i class="fa-fw fas fa-tags"></i>
			
			标签
		</a>
          
        </li>
        
        <li class="m-nav-item">
	  
		<a href="/categories" class="waves-effect waves-light">
			
			    <i class="fa-fw fas fa-bookmark"></i>
			
			分类
		</a>
          
        </li>
        
        <li class="m-nav-item">
	  
		<a href="/archives" class="waves-effect waves-light">
			
			    <i class="fa-fw fas fa-archive"></i>
			
			归档
		</a>
          
        </li>
        
        <li class="m-nav-item">
	  
		<a href="/about" class="waves-effect waves-light">
			
			    <i class="fa-fw fas fa-user-circle"></i>
			
			关于
		</a>
          
        </li>
        
        <li class="m-nav-item">
	  
		<a href="/contact" class="waves-effect waves-light">
			
			    <i class="fa-fw fas fa-comments"></i>
			
			留言板
		</a>
          
        </li>
        
        <li class="m-nav-item">
	  
		<a href="/friends" class="waves-effect waves-light">
			
			    <i class="fa-fw fas fa-address-book"></i>
			
			友情链接
		</a>
          
        </li>
        
        <li class="m-nav-item">
	  
		<a href="/navigate" class="waves-effect waves-light">
			
			    <i class="fa-fw fas fa-location-arrow"></i>
			
			导航
		</a>
          
        </li>
        
        
        <li><div class="divider"></div></li>
        <li>
            <a href="https://github.com/Aunean-ls" class="waves-effect waves-light" target="_blank">
                <i class="fab fa-github-square fa-fw"></i>Fork Me
            </a>
        </li>
        
    </ul>
</div>


        </div>

        
            <style>
    .nav-transparent .github-corner {
        display: none !important;
    }

    .github-corner {
        position: absolute;
        z-index: 10;
        top: 0;
        right: 0;
        border: 0;
        transform: scale(1.1);
    }

    .github-corner svg {
        color: #0f9d58;
        fill: #fff;
        height: 64px;
        width: 64px;
    }

    .github-corner:hover .octo-arm {
        animation: a 0.56s ease-in-out;
    }

    .github-corner .octo-arm {
        animation: none;
    }

    @keyframes a {
        0%,
        to {
            transform: rotate(0);
        }
        20%,
        60% {
            transform: rotate(-25deg);
        }
        40%,
        80% {
            transform: rotate(10deg);
        }
    }
</style>

<a href="https://github.com/Aunean-ls" class="github-corner tooltipped hide-on-med-and-down" target="_blank"
   data-tooltip="Fork Me" data-position="left" data-delay="50">
    <svg viewBox="0 0 250 250" aria-hidden="true">
        <path d="M0,0 L115,115 L130,115 L142,142 L250,250 L250,0 Z"></path>
        <path d="M128.3,109.0 C113.8,99.7 119.0,89.6 119.0,89.6 C122.0,82.7 120.5,78.6 120.5,78.6 C119.2,72.0 123.4,76.3 123.4,76.3 C127.3,80.9 125.5,87.3 125.5,87.3 C122.9,97.6 130.6,101.9 134.4,103.2"
              fill="currentColor" style="transform-origin: 130px 106px;" class="octo-arm"></path>
        <path d="M115.0,115.0 C114.9,115.1 118.7,116.5 119.8,115.4 L133.7,101.6 C136.9,99.2 139.9,98.4 142.2,98.6 C133.8,88.0 127.5,74.4 143.8,58.0 C148.5,53.4 154.0,51.2 159.7,51.0 C160.3,49.4 163.2,43.6 171.4,40.1 C171.4,40.1 176.1,42.5 178.8,56.2 C183.1,58.6 187.2,61.8 190.9,65.4 C194.5,69.0 197.7,73.2 200.1,77.6 C213.8,80.2 216.3,84.9 216.3,84.9 C212.7,93.1 206.9,96.0 205.4,96.6 C205.1,102.4 203.0,107.8 198.3,112.5 C181.9,128.9 168.3,122.5 157.7,114.1 C157.9,116.9 156.7,120.9 152.7,124.9 L141.0,136.5 C139.8,137.7 141.6,141.9 141.8,141.8 Z"
              fill="currentColor" class="octo-body"></path>
    </svg>
</a>
        
    </nav>

</header>

    
<script src="/libs/cryptojs/crypto-js.min.js"></script>
<script>
    (function() {
        let pwd = '';
        if (pwd && pwd.length > 0) {
            if (pwd !== CryptoJS.SHA256(prompt('请输入访问本文章的密码')).toString(CryptoJS.enc.Hex)) {
                alert('密码错误，将返回主页！');
                location.href = '/';
            }
        }
    })();
</script>




<div class="bg-cover pd-header post-cover" style="background-image: url('https://cdn.jsdelivr.net/gh/Aunean-ls/pic/img/u013.webp')">
    <div class="container" style="right: 0px;left: 0px;">
        <div class="row">
            <div class="col s12 m12 l12">
                <div class="brand">
                    <h1 class="description center-align post-title">Hadoop搭建</h1>
                </div>
            </div>
        </div>
    </div>
</div>




<main class="post-container content">

    
    <link rel="stylesheet" href="/libs/tocbot/tocbot.css">
<style>
    #articleContent h1::before,
    #articleContent h2::before,
    #articleContent h3::before,
    #articleContent h4::before,
    #articleContent h5::before,
    #articleContent h6::before {
        display: block;
        content: " ";
        height: 100px;
        margin-top: -100px;
        visibility: hidden;
    }

    #articleContent :focus {
        outline: none;
    }

    .toc-fixed {
        position: fixed;
        top: 64px;
    }

    .toc-widget {
        width: 345px;
        padding-left: 20px;
    }

    .toc-widget .toc-title {
        padding: 35px 0 15px 17px;
        font-size: 1.5rem;
        font-weight: bold;
        line-height: 1.5rem;
    }

    .toc-widget ol {
        padding: 0;
        list-style: none;
    }

    #toc-content {
        padding-bottom: 30px;
        overflow: auto;
    }

    #toc-content ol {
        padding-left: 10px;
    }

    #toc-content ol li {
        padding-left: 10px;
    }

    #toc-content .toc-link:hover {
        color: #42b983;
        font-weight: 700;
        text-decoration: underline;
    }

    #toc-content .toc-link::before {
        background-color: transparent;
        max-height: 25px;

        position: absolute;
        right: 23.5vw;
        display: block;
    }

    #toc-content .is-active-link {
        color: #42b983;
    }

    #floating-toc-btn {
        position: fixed;
        right: 15px;
        bottom: 76px;
        padding-top: 15px;
        margin-bottom: 0;
        z-index: 998;
    }

    #floating-toc-btn .btn-floating {
        width: 48px;
        height: 48px;
    }

    #floating-toc-btn .btn-floating i {
        line-height: 48px;
        font-size: 1.4rem;
    }
</style>
<div class="row">
    <div id="main-content" class="col s12 m12 l9">
        <!-- 文章内容详情 -->
<div id="artDetail">
    <div class="card">
        <div class="card-content article-info">
            <div class="row tag-cate">
                <div class="col s7">
                    
                    <div class="article-tag">
                        
                            <a href="/tags/Hadoop/">
                                <span class="chip bg-color">Hadoop</span>
                            </a>
                        
                    </div>
                    
                </div>
                <div class="col s5 right-align">
                    
                    <div class="post-cate">
                        <i class="fas fa-bookmark fa-fw icon-category"></i>
                        
                            <a href="/categories/%E5%AD%A6%E4%B9%A0%E7%AC%94%E8%AE%B0/" class="post-category">
                                学习笔记
                            </a>
                        
                    </div>
                    
                </div>
            </div>

            <div class="post-info">
                
                <div class="post-date info-break-policy">
                    <i class="far fa-calendar-minus fa-fw"></i>发布日期:&nbsp;&nbsp;
                    2021-04-19
                </div>
                

                
                <div class="post-date info-break-policy">
                    <i class="far fa-calendar-check fa-fw"></i>更新日期:&nbsp;&nbsp;
                    2021-09-30
                </div>
                

                
                <div class="info-break-policy">
                    <i class="far fa-file-word fa-fw"></i>文章字数:&nbsp;&nbsp;
                    7.7k
                </div>
                

                

                
            </div>
        </div>
        <hr class="clearfix">

        

        

        <div class="card-content article-card-content">
            <div id="articleContent">
                <h2 id="Hadoop配置文件讲解"><a href="#Hadoop配置文件讲解" class="headerlink" title="Hadoop配置文件讲解"></a>Hadoop配置文件讲解</h2><table>
<thead>
<tr>
<th>配置文件</th>
<th>功能描述</th>
</tr>
</thead>
<tbody><tr>
<td>hadoop-env.sh</td>
<td>配置 Hadoop 运行所需的环境变量</td>
</tr>
<tr>
<td>yarn-env.sh</td>
<td>配置 YARN 运行所需的环境变量</td>
</tr>
<tr>
<td>core-site.xml</td>
<td>Hadoop 核心全局配置文件，可在其他配置文件中引用改文件</td>
</tr>
<tr>
<td>hdfs-site.xml</td>
<td>HDFS 配置文件，继承 core-site.xml 配置文件</td>
</tr>
<tr>
<td>mapred-site.xml</td>
<td>MapReduce 配置文件，继承 core-site.xml 配置文件</td>
</tr>
<tr>
<td>yarn-site.xml</td>
<td>YARN 配置文件，继承 core-site.xml 配置文件</td>
</tr>
</tbody></table>
<p><a target="_blank" rel="noopener" href="https://hadoop.apache.org/docs/current/">官方文档</a></p>
<p><a target="_blank" rel="noopener" href="https://hadoop.apache.org/docs/r2.6.0/hadoop-project-dist/hadoop-common/DeprecatedProperties.html">2.6.0官方文档</a></p>
<p><a target="_blank" rel="noopener" href="https://hadoop.apache.org/docs/r2.7.2/hadoop-project-dist/hadoop-common/DeprecatedProperties.html">2.7.2官方文档</a></p>
<h3 id="1-core-site-xml-常用参数"><a href="#1-core-site-xml-常用参数" class="headerlink" title="1.core-site.xml 常用参数"></a>1.core-site.xml 常用参数</h3><pre class=" language-xml"><code class="language-xml"><span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>fs.defaultFS<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>hdfs://node:9000<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>HDFS的URI，设定namenode的主机名及端口<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>

<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>hadoop.tmp.dir<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>/usr/local/src/hadoop-2.7.2/tmp<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>节点上本地的hadoop临时文件夹，之前一定要先建立好<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>configuration</span><span class="token punctuation">></span></span>

<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>fs.trash.interval<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>1440<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>开启hdfs的垃圾桶机制，删除掉的数据可以从垃圾桶中回收，单位分钟<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>

<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>fs.trash.checkpoint.interval<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>0<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>

<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>io.file.buffer.size<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>4096<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>缓冲区大小，实际工作中根据服务器性能动态调整<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>

<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>fs.df.interval<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>5000<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>设置磁盘空间统计间隔为5秒<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>
</code></pre>
<h3 id="2-hdfs-site-xml-常用参数"><a href="#2-hdfs-site-xml-常用参数" class="headerlink" title="2.hdfs-site.xml 常用参数"></a>2.hdfs-site.xml 常用参数</h3><pre class=" language-xml"><code class="language-xml"><span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>dfs.namenode.name.dir<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>file:/usr/local/src/hdfs/name<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>namenode上存储hdfs名字空间元数据 <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>

<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>dfs.datanode.data.dir<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>file:/usr/local/src/hdfs/data<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>datanode上数据块的物理存储位置<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>

<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>dfs.replication<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>2<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>副本个数，默认是3,应小于datanode机器数量<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>

<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>dfs.replication.max<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>512<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>最大块复制<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>

<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>dfs.namenode.replication.min<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>1<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>最小块复制<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>

<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>dfs.blocksize<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>134217728<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>文件存储的block块大小128 MB<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>

<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>dfs.namenode.fs-limits.min-block-size<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>1048576<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>最小块大小（字节）<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>

<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>dfs.namenode.fs-limits.max-blocks-per-file<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>67108864<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>每个文件的最大块数<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>

<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>dfs.http.address<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>master:50070<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>这个是主节点的地址与端口<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>

<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>dfs.namenode.secondary.http-address<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>slave1:50090<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span> 
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>这个是SecondNameNode的地址与通信端口<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>
</code></pre>
<h3 id="3-mapred-site-xml-常用参数"><a href="#3-mapred-site-xml-常用参数" class="headerlink" title="3.mapred-site.xml 常用参数"></a>3.mapred-site.xml 常用参数</h3><pre class=" language-xml"><code class="language-xml"><span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>mapreduce.framework.name<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>yarn<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>指定我们mapreduce运行在yarn集群上面<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>

<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>mapreduce.task.io.sort.mb<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>128<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>排序文件的内存缓存大小为128M<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>

<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>mapreduce.job.maps<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>3<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>设置每个作业缺省的map任务数为3<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>
    
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>mapreduce.job.ubertask.enable<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>true<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>开启mapreduce的小任务模式，用于调优<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>


<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>mapreduce.jobhistory.address<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>node:10020<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>配置mapreduce的jobhistory，可以查看我们所有运行完成的任务的一些情况<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>

<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>mapreduce.jobhistory.webapp.address<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>node:19888<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>配置mapreduce的jobhistory的访问地址<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>
</code></pre>
<h3 id="4-yarn-site-xml-常用参数"><a href="#4-yarn-site-xml-常用参数" class="headerlink" title="4.yarn-site.xml 常用参数"></a>4.yarn-site.xml 常用参数</h3><pre class=" language-xml"><code class="language-xml"><span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>yarn.resourcemanager.hostname<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>node<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>指定resourcemanager所在的hostname，
    即指定yarn的老大即ResourceManger的地址<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>

<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>yarn.nodemanager.aux-services<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>mapreduce_shuffle<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>NodeManager上运行的附属服务。
    指定NodeManager获取数据的方式是shuffle
    需配置成mapreduce_shuffle，才可运行MapReduce程序<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>

<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>yarn.log-aggregation-enable<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>true<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>日志的聚合功能，方便我们查看任务执行完成之后的日志记录<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>
    
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>yarn.log-aggregation.retain-seconds<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>604800<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>聚合日志的保存时长，这里的值不要设置的太小<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>

<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>yarn.log.server.url<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>http://node:19888/jobhistory.logs<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>配置日志服务器的地址,node节点使用<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>

<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>yarn.nodemanager.log-dirs<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>    
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>/tmp/yarn/userlogs<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>存储container容器日志的位置<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>
</code></pre>
<h2 id="设置环境只对当前用户有效"><a href="#设置环境只对当前用户有效" class="headerlink" title="设置环境只对当前用户有效"></a>设置环境只对当前用户有效</h2><pre class=" language-shell"><code class="language-shell">vi .bash_profile

1.每次进入完全分布式有些命令未找到都必须要source一下
解决办法：
- 在~/.bashrc里面加一句source /etc/profile

hadoop dfsadmin -safemode leave 关闭安全模式
</code></pre>
<h2 id="各服务启动方式"><a href="#各服务启动方式" class="headerlink" title="各服务启动方式"></a>各服务启动方式</h2><p><strong>方式一：逐一启动</strong></p>
<pre class=" language-shell"><code class="language-shell">hadoop-daemon.sh start|stop namenode|datanode|journalnode
yarn-daemon.sh start|stop resourcemanager|nodemanager
</code></pre>
<p><strong>方式二：分开启动</strong></p>
<pre class=" language-shell"><code class="language-shell">start-dfs.sh
start-yarn.sh
</code></pre>
<p><strong>方式三：一起启动</strong></p>
<pre class=" language-shell"><code class="language-shell">start-all.sh
</code></pre>
<pre class=" language-shell"><code class="language-shell"># 启动zkfc节点
hadoop-daemon.sh start zkfc

# 启动Zookeeper节点
zkServer.sh start

# 查看NameNode和DataNode信息
hdfs dfsadmin -report

# 查看NameNode的节点状态
hdfs haadmin -getServiceState nn1

# 查看ResourceManager的节点状态
yarn rmadmin -getServiceState rm1
</code></pre>
<h2 id="配置yarn的历史服务器及日志产看功能"><a href="#配置yarn的历史服务器及日志产看功能" class="headerlink" title="配置yarn的历史服务器及日志产看功能"></a>配置yarn的历史服务器及日志产看功能</h2><pre class=" language-xml"><code class="language-xml">vi mapred-site.xml
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>mapreduce.jobhistory.address<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>node:10020<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>配置mapreduce的jobhistory，可以查看我们所有运行完成的任务的一些情况<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>

<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>mapreduce.jobhistory.webapp.address<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>node:19888<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>配置mapreduce的jobhistory的访问地址<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>


vi yarn-site.xml
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>yarn.log-aggregation-enable<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>true<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>

<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>yarn.log-aggregation.retain-seconds<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>604800<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>

<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>yarn.log.server.url<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>http://node:19888/jobhistory.logs<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>

# 启动 
mr-jobhistory-daemon.sh start historyserver
</code></pre>
<h2 id="伪分布式搭建"><a href="#伪分布式搭建" class="headerlink" title="伪分布式搭建"></a>伪分布式搭建</h2><h3 id="1-修改-IP-地址"><a href="#1-修改-IP-地址" class="headerlink" title="1.修改 IP 地址"></a>1.修改 IP 地址</h3><pre class=" language-shell"><code class="language-shell">vi /etc/sysconfig/network-scripts/ifcfg-ens33
</code></pre>
<h3 id="2-配置其中文件"><a href="#2-配置其中文件" class="headerlink" title="2.配置其中文件"></a>2.配置其中文件</h3><pre class=" language-shell"><code class="language-shell">TYPE=Ethernet
PROXY_METHOD=none
BROWSER_ONLY=no
BOOTPROTO=static
DEFROUTE=yes
IPV4_FAILURE_FATAL=no
IPV6INIT=yes
IPV6_AUTOCONF=yes
IPV6_DEFROUTE=yes
IPV6_FAILURE_FATAL=no
IPV6_ADDR_GEN_MODE=stable-privacy
NAME=ens33
UUID=7397575d-294f-44ad-88a6-7476a5f27106
DEVICE=ens33
ONBOOT=yes
IPADDR=192.168.136.220
NETMASK=255.255.255.0
GATEWAY=192.168.136.2
DNS1=119.29.29.29
</code></pre>
<h3 id="3-重启网络"><a href="#3-重启网络" class="headerlink" title="3.重启网络"></a>3.重启网络</h3><pre class=" language-shell"><code class="language-shell">systemctl restart network
</code></pre>
<h3 id="4-修改主机名"><a href="#4-修改主机名" class="headerlink" title="4.修改主机名"></a>4.修改主机名</h3><pre class=" language-shell"><code class="language-shell">查看主机名
hostname

修改主机名
hostnamectl set-hostname node0
</code></pre>
<h3 id="5-关闭防火墙"><a href="#5-关闭防火墙" class="headerlink" title="5.关闭防火墙"></a>5.关闭防火墙</h3><pre class=" language-shell"><code class="language-shell">关闭防火墙
systemctl stop firewalld

关闭防火墙自启动
systemctl disable firewalld

检查防火墙
systemctl status firewalld

关闭 selinux防火墙
vi /etc/sysconfig/selinux
把 SELINUX=enforcing 变为 SELINUX=disabled
</code></pre>
<h3 id="6-映射主机名和IP地址"><a href="#6-映射主机名和IP地址" class="headerlink" title="6.映射主机名和IP地址"></a>6.映射主机名和IP地址</h3><pre class=" language-shell"><code class="language-shell">vi /etc/hosts

192.168.136.150 master1-1
</code></pre>
<h3 id="7-安装-rz，使用rz上传所需文件"><a href="#7-安装-rz，使用rz上传所需文件" class="headerlink" title="7.安装 rz，使用rz上传所需文件"></a>7.安装 rz，使用rz上传所需文件</h3><pre class=" language-shell"><code class="language-shell">yum -y install lrzsz
</code></pre>
<h3 id="8-解压JDK-安装包到‘-usr-local-src’路径，并配置环境变量"><a href="#8-解压JDK-安装包到‘-usr-local-src’路径，并配置环境变量" class="headerlink" title="8.解压JDK 安装包到‘/usr/local/src’路径，并配置环境变量"></a>8.解压JDK 安装包到‘/usr/local/src’路径，并配置环境变量</h3><pre class=" language-shell"><code class="language-shell">tar -zxvf *.tar -C /usr/local/src

配置环境变量
vi /etc/profile
export JAVA_HOME=/usr/local/src/jdk1.8
export PATH=$PATH:$JAVA_HOME/bin

刷新环境变量
source /etc/profile

查看是否配置jdk成功
java -version
</code></pre>
<h3 id="9-解压hadoop安装包到-‘-usr-local-src’-路径，并配置环境变量"><a href="#9-解压hadoop安装包到-‘-usr-local-src’-路径，并配置环境变量" class="headerlink" title="9.解压hadoop安装包到 ‘/usr/local/src’ 路径，并配置环境变量"></a>9.解压hadoop安装包到 ‘/usr/local/src’ 路径，并配置环境变量</h3><pre class=" language-shell"><code class="language-shell">tar -zxvf *.tar -C /usr/local/src

配置环境变量
vi /etc/profile
export JAVA_HOME=/usr/local/src/jdk1.8
export HADOOP_HOME=/usr/local/src/hadoop-2.7.2
export PATH=$PATH:$JAVA_HOME/bin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin

刷新环境变量
source /etc/profile

查看是否配置jdk成功
hadoop version
</code></pre>
<h3 id="10-修改Hadoop目录下的配置文件"><a href="#10-修改Hadoop目录下的配置文件" class="headerlink" title="10.修改Hadoop目录下的配置文件"></a>10.修改Hadoop目录下的配置文件</h3><pre class=" language-xml"><code class="language-xml">vi hadoop-env.sh
export JAVA_HOME=/usr/local/src/jdk1.8

vi core-site.xml
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>configuration</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>fs.defaultFS<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>hdfs://node:9000<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>HDFS的URI，设定namenode的主机名及端口<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>hadoop.tmp.dir<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>/usr/local/src/hadoop-2.7.2/tmp<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>节点上本地的hadoop临时文件夹，之前一定要先建立好<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>configuration</span><span class="token punctuation">></span></span>

vi hdfs-site.xml
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>configuration</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>dfs.replication<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>1<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>副本个数，默认是3,应小于datanode机器数量<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>

<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>dfs.http.address<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>master:50070<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>这个是主节点的地址与端口<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>

<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>dfs.namenode.secondary.http-address<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>slave1:50090<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span> 
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>这个是SecondNameNode的地址与通信端口<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>



mv mapred-site.xml.template mapred-site.xml
vi mapred-site.xml
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>configuration</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>mapreduce.framework.name<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>yarn<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>指定mapreduce使用yarn框架<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>configuration</span><span class="token punctuation">></span></span>

vi yarn-site.xml
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>configuration</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>yarn.resourcemanager.hostname<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>master1-1<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>指定resourcemanager所在的hostname，即指定yarn的老大即ResourceManger的地址<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>yarn.nodemanager.aux-services<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>mapreduce_shuffle<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>NodeManager上运行的附属服务。指定NodeManager获取数据的方式是shuffle需配置成mapreduce_shuffle，才可运行MapReduce程序<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>configuration</span><span class="token punctuation">></span></span>
</code></pre>
<h3 id="11-配置完成后，格式化-DFS"><a href="#11-配置完成后，格式化-DFS" class="headerlink" title="11.配置完成后，格式化 DFS"></a>11.配置完成后，格式化 DFS</h3><pre class=" language-shell"><code class="language-shell">hdfs namenode -format
</code></pre>
<h3 id="12-创建shh密匙实现免密"><a href="#12-创建shh密匙实现免密" class="headerlink" title="12.创建shh密匙实现免密"></a>12.创建shh密匙实现免密</h3><pre class=" language-shell"><code class="language-shell"># 安装openssh，在centos7中ssh服务默认是已经被安装了的。通过命令rpm -qa | grep openssh 查看是否安装了ssh服务
yum install openssh

ssh-keygen -t rsa  # 然后按三次回车
cd /root/.ssh
ssh-copy-id master1-1  # 将本机的公钥复制到远程机器的authorized_key文件中，然后实现免密登录
more authorized_keys
</code></pre>
<h2 id="完全分布式搭建"><a href="#完全分布式搭建" class="headerlink" title="完全分布式搭建"></a>完全分布式搭建</h2><h3 id="1-配置三台IP"><a href="#1-配置三台IP" class="headerlink" title="1.配置三台IP"></a>1.配置三台IP</h3><pre class=" language-shell"><code class="language-shell">vi /etc/sysconfig/network-scripts/ifcfg-ens33
TYPE=Ethernet
PROXY_METHOD=none
BROWSER_ONLY=no
BOOTPROTO=static
DEFROUTE=yes
IPV4_FAILURE_FATAL=no
IPV6INIT=yes
IPV6_AUTOCONF=yes
IPV6_DEFROUTE=yes
IPV6_FAILURE_FATAL=no
IPV6_ADDR_GEN_MODE=stable-privacy
NAME=ens33
DEVICE=ens33
ONBOOT=yes
IPADDR=192.168.136.150
NETMASK=255.255.255.0
GATEWAY=192.168.136.2
DNS1=119.29.29.29

# 重启网络
systemctl restart network
</code></pre>
<h3 id="2-关闭三台防火墙"><a href="#2-关闭三台防火墙" class="headerlink" title="2.关闭三台防火墙"></a>2.关闭三台防火墙</h3><pre class=" language-shell"><code class="language-shell">systemctl stop firewalld  # 关闭防火墙
systemctl disable firewalld  # 关闭防火墙开机自启
systemctl status firewalld  # 检查防火墙

vi /etc/sysconfig/selinux   # 关闭selinux防火墙
SELINUX=disabled
</code></pre>
<h3 id="3-修改三台主机名和映射主机名和IP地址"><a href="#3-修改三台主机名和映射主机名和IP地址" class="headerlink" title="3.修改三台主机名和映射主机名和IP地址"></a>3.修改三台主机名和映射主机名和IP地址</h3><pre class=" language-shell"><code class="language-shell">hostnamectl set-hostname master1-1

vi /etc/hosts
192.168.136.150 master1-1
192.168.136.151 slave1-1
192.168.136.152 slave1-2
</code></pre>
<h3 id="4-安装-rz，使用rz上传所需文件"><a href="#4-安装-rz，使用rz上传所需文件" class="headerlink" title="4.安装 rz，使用rz上传所需文件"></a>4.安装 rz，使用rz上传所需文件</h3><pre class=" language-shell"><code class="language-shell">yum -y install lrzsz
</code></pre>
<h3 id="5-安装Hadoop和JDK，并配置好环境"><a href="#5-安装Hadoop和JDK，并配置好环境" class="headerlink" title="5.安装Hadoop和JDK，并配置好环境"></a>5.安装Hadoop和JDK，并配置好环境</h3><pre class=" language-shell"><code class="language-shell"># 使用 tar 命令解压
tar -zxvf *.gz -C /usr/local/src

vi /etc/profile
export JAVA_HOME=/usr/local/src/jdk1.8
export HADOOP_HOME=/usr/local/src/hadoop-2.7.2
PATH=$PATH:$JAVA_HOME/bin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin

# 刷新环境配置文件
source /etc/profile

# 查看Hadoop和JDK是否安装成功
java -version
hadoop version

# 可以先配置好主节点，然后使用 scp 命令分发给从节点，从而节约时间
scp -r /etc/profile slave1-1:/etc/
scp -r /etc/profile slave1-2:/etc/
</code></pre>
<h3 id="6-在三台同时同步时间"><a href="#6-在三台同时同步时间" class="headerlink" title="6.在三台同时同步时间"></a>6.在三台同时同步时间</h3><pre class=" language-shell"><code class="language-shell"># 安装NTP服务
yum install ntp
# 手动同步时间
ntpdate -u ntp1.aliyun.com
# 查看时间
date
</code></pre>
<h3 id="7-在三个节点删除原有目录，重新生成密匙对"><a href="#7-在三个节点删除原有目录，重新生成密匙对" class="headerlink" title="7.在三个节点删除原有目录，重新生成密匙对"></a>7.在三个节点删除原有目录，重新生成密匙对</h3><pre class=" language-shell"><code class="language-shell"># 删除原有.ssh目录
rm -rf /root/.ssh
# 重新生成迷密匙对，然后回车三次
ssh-keygen -t rsa
# 在各节点的.ssh目录下将公钥复制到主节点
cd /root/.ssh
ssh-copy-id master1-1
# 查看主节点的 authorized_keys 文件
more authorized_keys
# 将主节点上的 authorized_keys 文件远程拷贝到从节点。
scp -r authorized_keys slave1-1:~/.ssh/authorized_keys
</code></pre>
<h3 id="8-在主节点中配置Hadoop目录中的配置文件"><a href="#8-在主节点中配置Hadoop目录中的配置文件" class="headerlink" title="8.在主节点中配置Hadoop目录中的配置文件"></a>8.在主节点中配置Hadoop目录中的配置文件</h3><pre class=" language-xml"><code class="language-xml">cd /usr/local/src/hadoop-2.6.0/etc/hadoop

vi hadoop-env.sh
export JAVA_HOME=/usr/local/src/jdk1.8

vi core-site.xml
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>fs.defaultFS<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>hdfs://node:9000<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>HDFS的URI，设定namenode的主机名及端口<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>hadoop.tmp.dir<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>/usr/local/src/tmp<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>节点上本地的hadoop临时文件夹，之前一定要先建立好<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>

vi hdfs-site.xml
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>dfs.namenode.name.dir<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>file:/export/server/hdfs/name<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>namenode上存储hdfs名字空间元数据 <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>dfs.datanode.data.dir<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>file:/export/server/hdfs/data<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>datanode上数据块的物理存储位置<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>dfs.replication<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>2<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>副本个数，默认是3,应小于datanode机器数量<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>

vi mapred-env.sh
export JAVA_HOME=/usr/local/src/jdk1.8

vi mapred-site.xml
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>mapreduce.framework.name<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>yarn<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>指定mapreduce使用yarn框架<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>

vi yarn-env.sh
export JAVA_HOME=/usr/local/src/jdk1.8

vi yarn-site.xml
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>yarn.resourcemanager.hostname<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>node<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>指定resourcemanager所在的hostname，
    即指定yarn的老大即ResourceManger的地址<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>yarn.nodemanager.aux-services<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>mapreduce_shuffle<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>NodeManager上运行的附属服务。
    指定NodeManager获取数据的方式是shuffle
    需配置成mapreduce_shuffle，才可运行MapReduce程序<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>

vi slaves
slave1-1
slave1-2

# 将主节点上的配置文件分发到两个从节点
scp -r /usr/local/src/hadoop-2.7.2/etc/hadoop node1:/url/local/src/hadoop-2.7.2/etc/
scrp -r /usr/local/src/hadoop-2.7.2/etc/hadoop node2:/url/local/src/hadoop-2.7.2/etc/
</code></pre>
<h3 id="9-在主节点上格式化-HDFS"><a href="#9-在主节点上格式化-HDFS" class="headerlink" title="9.在主节点上格式化 HDFS"></a>9.在主节点上格式化 HDFS</h3><pre class=" language-shell"><code class="language-shell">hdfs namenode -format
</code></pre>
<h3 id="10-启动节点"><a href="#10-启动节点" class="headerlink" title="10.启动节点"></a>10.启动节点</h3><pre class=" language-shel"><code class="language-shel">start-all.sh
</code></pre>
<h2 id="zookeeper集群搭建"><a href="#zookeeper集群搭建" class="headerlink" title="zookeeper集群搭建"></a>zookeeper集群搭建</h2><h3 id="1-在完全分布式的基础上搭建"><a href="#1-在完全分布式的基础上搭建" class="headerlink" title="1.在完全分布式的基础上搭建"></a>1.在完全分布式的基础上搭建</h3><p>检查完全分布式集群，没有问题就可以直接搭建了</p>
<p>这里我们可以在主节点配置，然后分发给从节点，再修改从节点的myid</p>
<h3 id="2-解压到某个文件夹"><a href="#2-解压到某个文件夹" class="headerlink" title="2.解压到某个文件夹"></a>2.解压到某个文件夹</h3><pre class=" language-shell"><code class="language-shell">tar -zxvf zookeeper* -C /usr/local/src/
</code></pre>
<h3 id="3-进入到zookeeper文件夹"><a href="#3-进入到zookeeper文件夹" class="headerlink" title="3.进入到zookeeper文件夹"></a>3.进入到zookeeper文件夹</h3><pre class=" language-shell"><code class="language-shell">创建一个目录，这里叫 data
mkdir data
cd data
echo 1 > myid

cd conf
mv zoo_sample.cfg zoo.cfg    # 修改名字

vi zoo.cfg

tickTime=2000  # tickTime：Client和Server通信心跳数。单位：毫秒
# Zookeeper服务器之间或客户端与服务器之间维持心跳的时间间隔，也就是每隔tickTime的时间就会发送一个心跳。tickTime以毫秒为单位

initLimit=10  # LF初始通信时限。
# 集群中的follower服务器(F)与leader服务器(L)之间初始连接时能容忍的最多心跳数（tickTime的数量)。

syncLimit=5  # LF同步通信时限。
# 集群中的follower服务器与leader服务器之间请求和应答之间能容忍的最多心跳数（tickTime的数量)。

dataDir=/tmp/zookeeper  # 数据文件目录。
# Zookeeper保存数据的目录，默认情况下，Zookeeper将写数据的日志文件也保存在这个目录里。

clientPort=2181  # 客户端连接端口。
# 客户端连接Zookeeper服务器的端口，Zookeeper会监听这个端口，接受客户端的访问请求。

maxClientCnxns=60  # 客服端最大连接数
# 默认值是60，一个客户端能够连接到同一个服务器上的最大连接数，根据IP来区分。如果设置为0，表示没有任何限制。设置该值一方面是为了防止DoS攻击。

# Be sure to read the maintenance section of the
# administrator guide before turning on autopurge.
#
# http://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_maintenance
#
# The number of snapshots to retain in dataDir
#autopurge.snapRetainCount=3
# Purge task interval in hours
# Set to "0" to disable auto purge feature
#autopurge.purgeInterval=1
dataDir=/export/server/zookeeper-3.4.8/data
server.1=node:2888:3888
server.2=node1:2888:3888
server.3=node2:2888:3888
</code></pre>
<h3 id="4-配置环境变量"><a href="#4-配置环境变量" class="headerlink" title="4.配置环境变量"></a>4.配置环境变量</h3><pre class=" language-shell"><code class="language-shell">vi /etc/profile

export ZOOKEEPER_HOME=/usr/local/src/zookeeper-3.4.5

export ZOOKEEPER_HOME=/export/server/zookeeper-3.4.8
export PATH=$PATH:$ZOOKEEPER_HOME/bin
export CLASSPATH=.:$ZOOKEEPER_HOME/lib/dt.jar:$ZOOKEEPER_HOME/lib/tools.jar
</code></pre>
<h3 id="5-远程分发"><a href="#5-远程分发" class="headerlink" title="5.远程分发"></a>5.远程分发</h3><pre class=" language-shell"><code class="language-shell">scp -r /etc/profile node1:/etc/
scp -r /etc/profile node2:/etc/

scp -r /usr/local/src/zookeeper-3.4.5 node1:/usr/local/src/
scp -r /usr/local/src/zookeeper-3.4.5 node2:/usr/local/src/
</code></pre>
<h3 id="6-修改其他节点的myid"><a href="#6-修改其他节点的myid" class="headerlink" title="6.修改其他节点的myid"></a>6.修改其他节点的myid</h3><pre class=" language-shell"><code class="language-shell">cd /usr/local/src/zookeeper-3.4.5/data
vi myid
</code></pre>
<h2 id="高可用搭建"><a href="#高可用搭建" class="headerlink" title="高可用搭建"></a>高可用搭建</h2><h3 id="HDFS-HA-环境搭建"><a href="#HDFS-HA-环境搭建" class="headerlink" title="HDFS HA 环境搭建"></a>HDFS HA 环境搭建</h3><h4 id="1-在完成zookeeper的基础上搭建"><a href="#1-在完成zookeeper的基础上搭建" class="headerlink" title="1.在完成zookeeper的基础上搭建"></a>1.在完成zookeeper的基础上搭建</h4><h4 id="2-在完全分布式集群的主节点上修改配置文件"><a href="#2-在完全分布式集群的主节点上修改配置文件" class="headerlink" title="2.在完全分布式集群的主节点上修改配置文件"></a>2.在完全分布式集群的主节点上修改配置文件</h4><pre class=" language-shell"><code class="language-shell">cd /usr/local/src/hadoop-2.6.0/etc/hadoop

核心配置文件
core-site.xml -> 修改

HDFS配置文件
hadoop-env.sh -> 不变
hdfs-site.xml -> 修改

MapReduce配置文件
mapred-env.sh -> 不变
mapred-site.xml -> 不变

slaves -> 修改
</code></pre>
<pre class=" language-xml"><code class="language-xml">vi core-site.xml
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>fs.defaultFS<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>hdfs://mycluster<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>把两个NameNode的地址组装成一个集群mycluster<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>

# 这里的路径默认是NameNode、DataNode、JournalNode等存放数据的公共目录。用户也可以自己单独指定这三类节点的目录
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>hadoop.tmp.dir<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>/usr/local/src/hadoop-2.6.0/data/ha/tmp<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>指定hadoop运行时产生文件的存储目录<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>

<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>ha.zookeeper.quorum<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>node:2181,node1:2181,node2:2181<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>指定ZKFC故障自动切换转移<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>



vi hdfs-site.xml
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>dfs.replication<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>2<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>设置dsf副本数，默认3个<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>

<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>dfs.nameservices<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>mycluster<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>完全分布式集群名称<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>

<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>dfs.ha.namenodes.mycluster<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>nn1,nn2<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>集群中NameNode节点都有哪些<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>

<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>dfs.namenode.rpc-address.mycluster.nn1<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>node:8020<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>nn1的RPC通信地址<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>

<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>dfs.namenode.rpc-address.mycluster.nn2<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>node1:8020<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>nn2的RPC通信地址<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>

<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>dfs.namenode.http-address.mycluster.nn1<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>node:50070<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>nn1的http通信地址<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>

<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>dfs.namenode.http-address.mycluster.nn2<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>node1:50070<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>nn2的http通信地址<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>

<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>dfs.namenode.shared.edits.dir<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>qjournal://node:8485;node1:8485;node2:8485/mycluster<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>指定NameNode元数据在JournalNode上的存放位置<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>

<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>dfs.ha.fencing.methods<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>
    sshfence
    shell(/bin/true)
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>配置隔离机制，即同一时刻只能由一台服务器对外响应<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>

<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>dfs.ha.fencing.ssh.private-key-files<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>/root/.ssh/id_rsa<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>使用隔离机制时需要ssh无密钥登录<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>

<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>dfs.journalnode.edits.dir<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>/export/server/hadoop-2.6.0/data/ha/jn<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>声明journalnode服务器存储目录<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>

<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>dfs.permissions.enabled<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>false<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>关闭权限检查<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>

<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>dfs.client.failover.proxy.provider.mycluster<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>                                         <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>指定mycluster出现故障时，负责执行故障切换的类<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>

<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>dfs.ha.automatic-failover.enabled<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>true<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>配置自动故障转移<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>　


vi slaves
node
node1
node2
</code></pre>
<h4 id="3-将主节点的配置文件分发给从节点"><a href="#3-将主节点的配置文件分发给从节点" class="headerlink" title="3.将主节点的配置文件分发给从节点"></a>3.将主节点的配置文件分发给从节点</h4><pre class=" language-shell"><code class="language-shell"># 怎么分发要看所在文件夹位置和安装位置
scp -r ../hadoop node1:/$HADOOP_HOME/etc
scp -r hadoop node2:/usr/local/src/hadoop-2.6.0/etc
</code></pre>
<h4 id="4-关闭集群，启动-zookeeper"><a href="#4-关闭集群，启动-zookeeper" class="headerlink" title="4.关闭集群，启动 zookeeper"></a>4.关闭集群，启动 zookeeper</h4><pre class=" language-shell"><code class="language-shell"># 在主节点输入，服务关闭的就不用管了
stop-all.sh

# 在所有节点输入
zkServer.sh start
zkServer.sh status  # 查看状态，正确的状态只有一台机器时leader，其余为follower
# 用jpa查看的进程时QuorumPeerMain
</code></pre>
<h4 id="5-在主节点上格式化-zookeeper"><a href="#5-在主节点上格式化-zookeeper" class="headerlink" title="5.在主节点上格式化 zookeeper"></a>5.在主节点上格式化 zookeeper</h4><pre class=" language-shell"><code class="language-shell">hdfs zkfc -formatZK  # 目的是在Zookeeper集群上建立HA响应节点
</code></pre>
<h4 id="6-启动-JournalNode-集群"><a href="#6-启动-JournalNode-集群" class="headerlink" title="6.启动 JournalNode 集群"></a>6.启动 JournalNode 集群</h4><pre class=" language-shell"><code class="language-shell"># 在所有节点输入
hadoop-daemon.sh start journalnode  # 启动JournalNode是为了创建/data/ha/jn,此时jn里面是空的
</code></pre>
<h4 id="7-格式化NameNode"><a href="#7-格式化NameNode" class="headerlink" title="7.格式化NameNode"></a>7.格式化NameNode</h4><pre class=" language-shell"><code class="language-shell"># 格式化集群的一个NameNode节点，可以在nn1和nn2任选一个格式化。这里选nn1即主节点
hdfs namenode -format
# 格式化NameNode，此里面jn会产生集群等ID等信息，/data/ha/tmp也会产生相关信息。
</code></pre>
<h4 id="8-启动-NameNode，并同步到备用-NameNode"><a href="#8-启动-NameNode，并同步到备用-NameNode" class="headerlink" title="8.启动 NameNode，并同步到备用 NameNode"></a>8.启动 NameNode，并同步到备用 NameNode</h4><pre class=" language-shell"><code class="language-shell"># 在nn1即node（主节点）节点单独启动namenode进程：
hadoop-daemon.sh start namenode

# 将格式化的namenode节点信息同步到备用的NameNode（nn2即node1节点），并启动namenode进程：
# 在 node1 节点输入
hdfs namenode -bootstrapStandby
hadoop-daemon.sh start namenode
</code></pre>
<h4 id="9-启动所有的-DataNode"><a href="#9-启动所有的-DataNode" class="headerlink" title="9.启动所有的 DataNode"></a>9.启动所有的 DataNode</h4><pre class=" language-shell"><code class="language-shell"># DataNode 是在 slaves 文件中配置的，所以在 nn1 即 node 节点上启动所有的 datanode 进程。
hadoop-daemons.sh start datanode
</code></pre>
<h4 id="10-启动-ZKFailoverCotroller"><a href="#10-启动-ZKFailoverCotroller" class="headerlink" title="10.启动 ZKFailoverCotroller"></a>10.启动 ZKFailoverCotroller</h4><pre class=" language-shell"><code class="language-shell"># 在 node 和 node1 上分别启动 ZkFailoverCotroller（主备切换控制器），
# 是 NameNode 机器 上一个独立的进程（进程名为 zkfc）。
hadoop-daemon.sh start zkfc
</code></pre>
<h4 id="11-到此就可以使用-jps-命令查看进程"><a href="#11-到此就可以使用-jps-命令查看进程" class="headerlink" title="11.到此就可以使用 jps 命令查看进程"></a>11.到此就可以使用 jps 命令查看进程</h4><pre class=" language-shell"><code class="language-shell">node  六个
node1 六个
node2 四个

查看NameNode的节点状态
hdfs haadmin -getServiceState nn1
hdfs haadmin -getServiceState nn2

关于故障转移机制无效，下载插件
yum install psmisc
</code></pre>
<p><img src="https://cdn.jsdelivr.net/gh/Aunean-ls/pic/img/2-8.png" alt="2-8"></p>
<h3 id="Yarn-HA-环境搭建"><a href="#Yarn-HA-环境搭建" class="headerlink" title="Yarn HA 环境搭建"></a>Yarn HA 环境搭建</h3><h4 id="1-在完成-HDFS-HA-的基础上安装"><a href="#1-在完成-HDFS-HA-的基础上安装" class="headerlink" title="1.在完成 HDFS HA 的基础上安装"></a>1.在完成 HDFS HA 的基础上安装</h4><pre><code>关闭所有进程。
</code></pre>
<h4 id="2-同样在hadoop目录下修改配置文件"><a href="#2-同样在hadoop目录下修改配置文件" class="headerlink" title="2.同样在hadoop目录下修改配置文件"></a>2.同样在hadoop目录下修改配置文件</h4><pre class=" language-xml"><code class="language-xml">vi yarn-site.xml

<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>yarn.nodemanager.aux-services<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>mapreduce_shuffle<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>NodeManager上运行的附属服务。
    指定NodeManager获取数据的方式是shuffle
    需配置成mapreduce_shuffle，才可运行MapReduce程序<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>

<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>yarn.resourcemanager.ha.enabled<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>true<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>启用resourcemanager ha<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>

<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>yarn.resourcemanager.cluster-id<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>rmCluster<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>指定resourcemanager的cluster-id<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>

<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>yarn.resourcemanager.ha.rm-ids<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>rm1,rm2<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>声明两台resourcemanager的名字<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>

<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>yarn.resourcemanager.hostname.rm1<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>node<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>指定第一台resourcemanager的地址<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>

<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>yarn.resourcemanager.hostname.rm2<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>node1<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>指定第二台resourcemanager的地址<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>

<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>yarn.resourcemanager.zk-address<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>node:2181,node1:2181,node2:2181<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>指定zookeeper集群的地址<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>

<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>yarn.resourcemanager.recovery.enabled<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>true<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>启用自动恢复<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>

<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>yarn.resourcemanager.store.class<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>指定resourcemanager的状态信息存储在zookeeper集群地址<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>
</code></pre>
<h4 id="3-远程分发"><a href="#3-远程分发" class="headerlink" title="3.远程分发"></a>3.远程分发</h4><pre class=" language-shell"><code class="language-shell">scp -r yarn-site.xml node1:/usr/local/src/hadoop-2.6.0/etc/hadoop/
scp -r yarn-site.xml node2:/usr/local/src/hadoop-2.6.0/etc/hadoop/

# 查看两个ResourceManager的节点状态
yarn rmadmin -getServiceState rm1
yarn rmadmin -getServiceState rm2

yarn rmadmin -transitionToStandby rm1

port:8088
</code></pre>
<h4 id="4-启动-Hadoop-HA-集群"><a href="#4-启动-Hadoop-HA-集群" class="headerlink" title="4.启动 Hadoop HA 集群"></a>4.启动 Hadoop HA 集群</h4><p>(1).在三个节点上启动 ZooKeeper。zkServer.sh start</p>
<p>(2).在 node 和 node1 上启动 zkfc。hadoop-daemon.sh start zkfc</p>
<p>(3).在 node 上启动 hadoop 集群。start-all.sh</p>
<p>(4).在 node1 上启动 resourcemanager。yarn-daemon.sh start resourcemanager</p>
<p><strong>注：全部启动后。节点数分别为</strong> <strong>8 8 5</strong></p>
<h2 id="MySQL的安装部署"><a href="#MySQL的安装部署" class="headerlink" title="MySQL的安装部署"></a>MySQL的安装部署</h2><h3 id="1-安装-wget"><a href="#1-安装-wget" class="headerlink" title="1.安装 wget"></a>1.安装 wget</h3><pre class=" language-shell"><code class="language-shell">yum -y install wget
</code></pre>
<h3 id="2-下载-rpm-安装文件"><a href="#2-下载-rpm-安装文件" class="headerlink" title="2.下载 rpm 安装文件"></a>2.下载 rpm 安装文件</h3><pre class=" language-shell"><code class="language-shell">wget http://repo.mysql.com/mysql-community-release-el7-7.noarch.rpm
</code></pre>
<h3 id="3-安装下载的-mysql-community-release-el7-7-noarch-rpm-文件"><a href="#3-安装下载的-mysql-community-release-el7-7-noarch-rpm-文件" class="headerlink" title="3.安装下载的 mysql-community-release-el7-7.noarch.rpm 文件"></a>3.安装下载的 mysql-community-release-el7-7.noarch.rpm 文件</h3><pre class=" language-shell"><code class="language-shell"># 安装下载的 mysql-community-release-el7-7.noarch.rpm 文件
rpm -ivh mysql-community-release-el7-7.noarch.rpm
</code></pre>
<h4 id="4-配置-MySQL-安装源信息"><a href="#4-配置-MySQL-安装源信息" class="headerlink" title="4.配置 MySQL 安装源信息"></a>4.配置 MySQL 安装源信息</h4><pre class=" language-shell"><code class="language-shell">vi /etc/yum.repos.d/mysql-community.repo
启用5.7的版本
# enabled 是否启用 1 启用，0 关闭
# gpgchenk 是否检查
</code></pre>
<h3 id="5-进行-MySQL-安装"><a href="#5-进行-MySQL-安装" class="headerlink" title="5.进行 MySQL 安装"></a>5.进行 MySQL 安装</h3><pre class=" language-shell"><code class="language-shell">yum install mysql-community-server
</code></pre>
<h3 id="6-配置MySQL"><a href="#6-配置MySQL" class="headerlink" title="6.配置MySQL"></a>6.配置MySQL</h3><pre class=" language-shell"><code class="language-shell"># 启用mysql
systemctl start mysqld

# 设为开机自启
systemctl enable mysqld
systemctl daemon-reload
</code></pre>
<h3 id="7-查看-mysql-5-7-数据库初始化root密码"><a href="#7-查看-mysql-5-7-数据库初始化root密码" class="headerlink" title="7.查看 mysql 5.7 数据库初始化root密码"></a>7.查看 mysql 5.7 数据库初始化root密码</h3><pre class=" language-shell"><code class="language-shell">grep 'temporary password' /var/log/mysqld.log

mysql -uroot -p
# 回车，然后把复制的密码粘贴进去
</code></pre>
<h3 id="8-登录后修改MySQL密码"><a href="#8-登录后修改MySQL密码" class="headerlink" title="8.登录后修改MySQL密码"></a>8.登录后修改MySQL密码</h3><pre class=" language-shell"><code class="language-shell">set global validate_password_policy=LOW;
set global validate_password_length=6;
alter user 'root'@'localhost' identified by '123456';
</code></pre>
<h3 id="9-开放远程连接"><a href="#9-开放远程连接" class="headerlink" title="9.开放远程连接"></a>9.开放远程连接</h3><pre class=" language-shell"><code class="language-shell"># 进入 mysql 数据库
use mysql;

update user set host='%' where user='root' and host='localhost';
</code></pre>
<h3 id="10-刷新设置"><a href="#10-刷新设置" class="headerlink" title="10.刷新设置"></a>10.刷新设置</h3><pre class=" language-shell"><code class="language-shell">flush privileges;
</code></pre>
<h2 id="hive的安装部署"><a href="#hive的安装部署" class="headerlink" title="hive的安装部署"></a>hive的安装部署</h2><h3 id="1-解压"><a href="#1-解压" class="headerlink" title="1.解压"></a>1.解压</h3><pre class=" language-shell"><code class="language-shell">tar -zxvf apache-hive* -C /usr/local/src/
</code></pre>
<h3 id="2-进入hive目录并改名"><a href="#2-进入hive目录并改名" class="headerlink" title="2.进入hive目录并改名"></a>2.进入hive目录并改名</h3><h3 id="3-修改环境配置"><a href="#3-修改环境配置" class="headerlink" title="3.修改环境配置"></a>3.修改环境配置</h3><pre class=" language-shell"><code class="language-shell"># 使环境变量只对当前用户生效
vi .bash_profile
export HIVE_HOME=/usr/local/src/hive
export PATH=$PATH:$HIVE_HOME/bin
source .bash_profile

# 对所有用户有效
vi /etc/profile
export HIVE_HOME=/usr/local/src/hive
export PATH=$PATH:$HIVE_HOME/bin
source /etc/profile
</code></pre>
<h3 id="4-修改hive配置文件"><a href="#4-修改hive配置文件" class="headerlink" title="4.修改hive配置文件"></a>4.修改hive配置文件</h3><pre class=" language-xml"><code class="language-xml"># 进入到hive配置文件目录
cd /usr/local/src/hive/conf

cp hive-env.sh.tamplate hive-env.sh
vi hive-env.sh
export JAVA_HOME=/usr/local/src/jdk1.8
export HADOOP_HOME=/usr/local/src/hadoop-2.6.0
export HIVE_HOME=/usr/local/src/hive

vi hive-site.xml    # 这个配置文件是我们自己创建的
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>configuration</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
    <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>javax.jdo.option.ConnectionURL<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
     <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>jdbc:mysql://node:3306/hiveDB?createDatabaseIfNotExist=true<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
     <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>数据库连接地址<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
     <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>javax.jdo.option.ConnectionDriverName<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
     <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>com.mysql.jdbc.Driver<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
     <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>数据库连接驱动<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
     <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>javax.jdo.option.ConnectionUserName<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
     <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>root<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
     <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>数据库连接用户名<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
     <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>javax.jdo.option.ConnectionPassword<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
     <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>123456<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
     <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>数据库连接密码<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>configuration</span><span class="token punctuation">></span></span>
</code></pre>
<h3 id="5-将MySQL数据库JDBC驱动拷贝到Hive安装目录的lib下"><a href="#5-将MySQL数据库JDBC驱动拷贝到Hive安装目录的lib下" class="headerlink" title="5.将MySQL数据库JDBC驱动拷贝到Hive安装目录的lib下"></a>5.将MySQL数据库JDBC驱动拷贝到Hive安装目录的lib下</h3><p><a target="_blank" rel="noopener" href="https://www.jianguoyun.com/p/DRD10aMQ3ovMCRiR8YME">驱动下载</a></p>
<h3 id="6-jar包修改"><a href="#6-jar包修改" class="headerlink" title="6.jar包修改"></a>6.jar包修改</h3><pre class=" language-shell"><code class="language-shell"># 将 hive 的 bin 目录下的 jline-2.12.jar 文件发送到 hadoop 的 share/hadoop/yarn/lib/ 目录下
cp jline-2.12.jar /usr/local/src/hadoop-2.6.0/share/hadoop/yarn/lib/

# 删除 hadoop 中的 jline-0.9.94.jar 文件
cd /usr/local/src/hadoop-2.6.0/share/hadoop/yarn/lib/
rm -rf jline-0.9.94.jar
</code></pre>
<h3 id="7-初始化"><a href="#7-初始化" class="headerlink" title="7.初始化"></a>7.初始化</h3><pre class=" language-shell"><code class="language-shell"># 在hive的bin目录下进行初始化
schematool -dbType mysql -initSchema

# 启动集群
start-all.sh

# 打开hive
hive

# 若是出现safemode的报错而导致进入不了hive，可以输入
hdfs dfsadmin -safemode leave
</code></pre>
<h2 id="sqoop的安装部署"><a href="#sqoop的安装部署" class="headerlink" title="sqoop的安装部署"></a>sqoop的安装部署</h2><p><a target="_blank" rel="noopener" href="https://sqoop.apache.org/docs/1.4.7/SqoopUserGuide.html">官方文档</a></p>
<h3 id="一、sqoop的安装"><a href="#一、sqoop的安装" class="headerlink" title="一、sqoop的安装"></a>一、sqoop的安装</h3><p>==<strong>sqoop要配合MySQL或其他数据库使用</strong>==</p>
<p>==<strong>这里配合MySQL使用。要开放MySQL远程连接。</strong>==</p>
<h4 id="1-解压、改名"><a href="#1-解压、改名" class="headerlink" title="1.解压、改名"></a>1.解压、改名</h4><pre class=" language-shell"><code class="language-shell">tar -zxvf sqoop* -C /usr/local/src/
</code></pre>
<h4 id="2-配置环境变量"><a href="#2-配置环境变量" class="headerlink" title="2.配置环境变量"></a>2.配置环境变量</h4><pre class=" language-shell"><code class="language-shell">vi /etc/profile
export SQOOP_HOME=/usr/local/src/sqoop-1.4
export PATH=$PATH:$SQOOP_HOME/bin
source /etc/profile

vi .bash_profile
export SQOOP_HOME=/usr/local/src/sqoop-1.4
export PATH=$PATH:$SQOOP_HOME/bin
source .bash_profile
</code></pre>
<h4 id="3-验证"><a href="#3-验证" class="headerlink" title="3.验证"></a>3.验证</h4><pre class=" language-shell"><code class="language-shell">sqoop version
</code></pre>
<h4 id="4-拷贝数据库驱动到-sqoop-的-lib-目录下"><a href="#4-拷贝数据库驱动到-sqoop-的-lib-目录下" class="headerlink" title="4.拷贝数据库驱动到 sqoop 的 lib 目录下"></a>4.拷贝数据库驱动到 sqoop 的 lib 目录下</h4><p><a target="_blank" rel="noopener" href="https://www.jianguoyun.com/p/DRD10aMQ3ovMCRiR8YME">驱动下载</a></p>
<h4 id="5-配置-sqoop-env-sh"><a href="#5-配置-sqoop-env-sh" class="headerlink" title="5.配置 sqoop-env.sh"></a>5.配置 sqoop-env.sh</h4><ul>
<li>改名</li>
</ul>
<pre class=" language-shell"><code class="language-shell">vi sqoop-env.sh

export HADOOP_COMMON_HOME=/usr/local/src/hadoop-2.6.0
export HADOOP_MAPRED_HOME=/usr/local/src/hadoop-2.6.0
export ZOOCFGDIR=/usr/local/src/zookeeper-3.4.5/conf
export ZOOKEEPER_HOME=/usr/local/src/zookeeper-3.4.5
export HIVE_HOME=/usr/local/src/hive
export HIVE_CONF_DIR=/usr/local/src/hive/conf
export HBASE_HOME=/usr/local/src/hbase
</code></pre>
<ul>
<li>验证： sqoop help</li>
</ul>
<h3 id="二、sqoop的使用"><a href="#二、sqoop的使用" class="headerlink" title="二、sqoop的使用"></a>二、sqoop的使用</h3><p><a target="_blank" rel="noopener" href="https://sqoop.apache.org/docs/1.4.7/SqoopUserGuide.html">官方文档</a></p>
<ul>
<li>公用参数：数据库连接</li>
</ul>
<table>
<thead>
<tr>
<th>参数</th>
<th>说明</th>
</tr>
</thead>
<tbody><tr>
<td>-connect</td>
<td>连接关系型数据库的URL</td>
</tr>
<tr>
<td>-connection-manager</td>
<td>指定要使用的连接管道类</td>
</tr>
<tr>
<td>-driver</td>
<td>Hadoop的根目录</td>
</tr>
<tr>
<td>-help</td>
<td>打印帮助信息</td>
</tr>
<tr>
<td>-password</td>
<td>数据库连接密码</td>
</tr>
<tr>
<td>-username</td>
<td>数据库的连接用户名</td>
</tr>
<tr>
<td>-verbose</td>
<td>在控制台打印出详细信息</td>
</tr>
</tbody></table>
<ul>
<li>公用参数：import</li>
</ul>
<table>
<thead>
<tr>
<th>参数</th>
<th>说明</th>
</tr>
</thead>
<tbody><tr>
<td>-enclosed-by <char></char></td>
<td>给字段值前加上指定的字符。</td>
</tr>
<tr>
<td>-escaped-by <char></char></td>
<td>对字段中的双引号加转义符。</td>
</tr>
<tr>
<td>-fields-terminated-by <char></char></td>
<td>设定每个字段是以什么符号为结束，默认为逗号。</td>
</tr>
<tr>
<td>-lines-terminated-by <char></char></td>
<td>设定每行记录之间的分隔符，默认是 \n。</td>
</tr>
<tr>
<td>-mysql-delimiters</td>
<td>MySQL默认的分隔符设置，字段之间以逗号分隔，行之间以 \n 分隔，默认转义符是 \ ，字段值以单引号包裹。</td>
</tr>
<tr>
<td>-optionally-enclosed-by <char></char></td>
<td>给带有双引号或单引号的字段值前后加上指定字符。</td>
</tr>
</tbody></table>
<ul>
<li>公用参数：export</li>
</ul>
<table>
<thead>
<tr>
<th>参数</th>
<th>说明</th>
</tr>
</thead>
<tbody><tr>
<td>-input-enclosed-by <char></char></td>
<td>对字段值前后加上指定字符</td>
</tr>
<tr>
<td>–input-escaped-by <char></char></td>
<td>对含有转义字符的字段做转义处理</td>
</tr>
<tr>
<td>–input-fields-terminated-by <char></char></td>
<td>字段之间的分隔符</td>
</tr>
<tr>
<td>–input-lines-terminated-by <char></char></td>
<td>行之间的分隔符</td>
</tr>
<tr>
<td>–input-optionally-enclosed-by <char></char></td>
<td>给带有双引号或单引号的字段前后加上指定字符</td>
</tr>
</tbody></table>
<ul>
<li>公用参数：hive</li>
</ul>
<table>
<thead>
<tr>
<th>参数</th>
<th>说明</th>
</tr>
</thead>
<tbody><tr>
<td>–hive-delims-replacement <arg></arg></td>
<td>用自定义的字符串替换掉数据中的\r\n和\013 \010等字符</td>
</tr>
<tr>
<td>-hive-drop-import-delims</td>
<td>在导入数据到hive时，去掉数据中的\r\n\013\010这样的字符</td>
</tr>
<tr>
<td>–map-column-hive <arg></arg></td>
<td>生成hive表时，可以更改生成字段的数据类型</td>
</tr>
<tr>
<td>–hive-partition-key</td>
<td>创建分区，后面直接跟分区名，分区字段的默认类型为string</td>
</tr>
<tr>
<td>–hive-partition-value <v></v></td>
<td>导入数据时，指定某个分区的值</td>
</tr>
<tr>
<td>–hive-home <dir></dir></td>
<td>hive的安装目录，可以通过该参数覆盖之前默认配置的目录</td>
</tr>
<tr>
<td>–hive-import</td>
<td>将数据从关系数据库中导入到hive表中</td>
</tr>
<tr>
<td>–hive-overwrite</td>
<td>覆盖掉在hive表中已经存在的数据</td>
</tr>
<tr>
<td>–create-hive-table</td>
<td>默认是false，即，如果目标表已经存在了，那么创建任务失败</td>
</tr>
<tr>
<td>–hive-table</td>
<td>后面接要创建的hive表,默认使用MySQL的表名</td>
</tr>
<tr>
<td>-table</td>
<td>指定关系数据库的表名</td>
</tr>
</tbody></table>
<h4 id="1-测试连接到MySQL数据库，列出MySQL中的所有数据库"><a href="#1-测试连接到MySQL数据库，列出MySQL中的所有数据库" class="headerlink" title="1.测试连接到MySQL数据库，列出MySQL中的所有数据库"></a>1.测试连接到MySQL数据库，列出MySQL中的所有数据库</h4><pre class=" language-shell"><code class="language-shell"># -
sqoop list-databases --connect jdbc:mysql://node:3306/ --username root -password 123456

sqoop list-tables --connect jdbc:mysql://walker:3306/spark --username root -password qaz3357375

# 查看dsj库下的所有表
sqoop list-tables --connect jdbc:mysql://node:3306/dsj --username root -password 123456
</code></pre>
<h4 id="2-在MySQL中建库、建表"><a href="#2-在MySQL中建库、建表" class="headerlink" title="2.在MySQL中建库、建表"></a>2.在MySQL中建库、建表</h4><pre class=" language-mysql"><code class="language-mysql">create database dsj;

create table t_daochu(
    id int not null primary key,
    name varchar(8) not null,
    sex varchar(8) not null,
    age int
    );
create table uv_info(
    id int not null primary key,
    name varchar(8) not null,
    sex varchar(8) not null,
    age int
    );

insert into t_daochu values(1,'xiaohong','girl',11),(2,'xiaoli','girl',12),(3,'xiaoming','boy',12),(4,'xiaojun','boy',14);
</code></pre>
<h4 id="3-mysql-数据-导入、导出-hdfs"><a href="#3-mysql-数据-导入、导出-hdfs" class="headerlink" title="3.mysql 数据 导入、导出 hdfs"></a>3.mysql 数据 导入、导出 hdfs</h4><pre class=" language-shell"><code class="language-shell">例题1：
# 使用sqoop将表中数据导入到HDFS。
sqoop import --connect jdbc:mysql://node:3306/dsj --username root --table t_daochu -password 123456 --target-dir /sqoop -m 1
# jdbc:mysql 驱动
# node:3306 主机名:端口
# --target-dir /sqoop 目标文件
# -m 1  map任务数
# sqoop import 或 sqoop-import

# sqoop-export 操作与 sqoop-import 的操作是相反的，即把 HDFS、hive、hbase 中的文件或数据导出到 RDBMS 数据库中，RDBMS 表必须存在，否则 sqoop-export 操作执行出错。
例题2：
将例题1中的文件  part-m-00000 导出到 MySQL 数据库 t_daochu 表。
sqoop export --connect jdbc:mysql://node:3306/dsj --table uv_info --columns "id,name,sex,age" --username root -password 123456 --export-dir /sqoop
</code></pre>
<h4 id="4-查看"><a href="#4-查看" class="headerlink" title="4.查看"></a>4.查看</h4><p><img src="C:\Users\Aunean\AppData\Roaming\Typora\typora-user-images\image-20201130214119569.png" alt="image-20201130214119569"></p>
<p><img src="C:\Users\Aunean\AppData\Roaming\Typora\typora-user-images\image-20201130214210459.png" alt="image-20201130214210459"></p>
<ul>
<li>例题二，先删除我们不需要的文件</li>
</ul>
<p><img src="C:\Users\Aunean\AppData\Roaming\Typora\typora-user-images\image-20201130220116651.png" alt="image-20201130220116651"></p>
<p><img src="C:\Users\Aunean\AppData\Roaming\Typora\typora-user-images\image-20201130220621277.png" alt="image-20201130220621277"></p>
<h4 id="5-关系型数据的表结构复制到-hive-中"><a href="#5-关系型数据的表结构复制到-hive-中" class="headerlink" title="5.关系型数据的表结构复制到 hive 中"></a>5.关系型数据的表结构复制到 hive 中</h4><pre class=" language-shell"><code class="language-shell">sqoop create-hive-table --connect jdbc:mysql://node1:3306/company --table staff --username root --password 123456 --hive-table test
</code></pre>
<ul>
<li>出现了报错，解决方法</li>
</ul>
<pre><code># 第一种方法。在 /etc/profile 中写入
export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:$HIVE_HOME/lib/*

# 第二种方法。将hive 里面的lib下的hive-exec-**.jar 放到sqoop 的lib 下
</code></pre>
<h4 id="6-关系型数据库和hive的使用"><a href="#6-关系型数据库和hive的使用" class="headerlink" title="6.关系型数据库和hive的使用"></a>6.关系型数据库和hive的使用</h4><pre class=" language-shell"><code class="language-shell"># 将MySQL dsj库中 t表中的数据传入到hive中
sqoop import --connect jdbc:mysql://node:3306/dsj --username root --password 123456 --table t_daochu --hive-import --hive-table dsj.t_daoru --fields-terminated-by '\t'

sqoop import --connect jdbc:mysql://node:3306/spark --username root --password 123456 --table person --hive-import --hive-table person --fields-terminated-by '\t'

# 将hive中的数据传入到MySQL中
sqoop export --connect jdbc:mysql://node:3306/dsj --username root --password 123456 --table uv_info --export-dir /user/hive/warehouse/t --input-fields-terminated-by '\t'

sqoop export --connect jdbc:mysql://node:3306/jobdata --username root --password 123456 --table t_city_detail --export-dir /user/hive/warehouse/jobdata.db/t_city_detail --input-fields-terminated-by '\001'
</code></pre>
<h4 id="注："><a href="#注：" class="headerlink" title="注："></a>注：</h4><ul>
<li>每个 RDBMS 表中只有一个单独列作为主键，即不能是多个列 R合键作为主键。</li>
<li>执行导入操作时，每个 RDBMS 表的所有列列都将被导入到 HDFS 上。</li>
<li>不能在 RDBMS 表上附加任何诸如 WHERE 条件的子句。</li>
</ul>
<h4 id="1-1-导入数据"><a href="#1-1-导入数据" class="headerlink" title="1.1 导入数据"></a>1.1 导入数据</h4><h5 id="1-1-1-RDBMS-到-HDFS"><a href="#1-1-1-RDBMS-到-HDFS" class="headerlink" title="1.1.1 RDBMS 到 HDFS"></a>1.1.1 RDBMS 到 HDFS</h5><pre class=" language-shell"><code class="language-shell">create database company;
create table company.staff(
id int(4) primary key not null auto_increment,  
name varchar(255), 
sex varchar(255)
);
insert into company.staff(name, sex) values('Thomas', 'Male');
insert into company.staff(name, sex) values('Catalina', 'FeMale');

# 全部导入
sqoop import --connect jdbc:mysql://master:3306/company --username root --password 123456 --table staff --target-dir /usr/company --delete-target-dir -m 1 --fields-terminated-by '\t'

# 查询导入
sqoop import \
--connect jdbc:mysql://node1:3306/company \
--username root \
--password 123456 \
--target-dir /user/company \
--delete-target-dir \
--num-mappers 1 \
--fields-terminated-by "\t" \
--query 'select name,sex from staff where id <=1 and $CONDITIONS;'

sqoop import \
--connect jdbc:mysql://node1:3306/company \
--username root \
--password 123456 \
--target-dir /user/company \
--delete-target-dir \
--num-mappers 1 \
--fields-terminated-by "\t" \
--query "select name,sex from staff where id <=2 and \$CONDITIONS;"

# 导入指定列
sqoop import \
--connect jdbc:mysql://node1:3306/company \
--username root \
--password 123456 \
--table staff \
--columns id,sex \
--target-dir /user/company \
--delete-target-dir \
--num-mappers 1 \
--fields-terminated-by "\t"
# 提示：columns 中如果涉及到多列，用逗号分隔，分隔时不要添加空格

# 使用 sqoop 关键字筛选查询导入数据
sqoop import \
--connect jdbc:mysql://node1:3306/company \
--username root \
--password 123456 \
--target-dir /user/company \
--delete-target-dir \
--m 1 \
--fields-terminated-by '\t' \
--table staff \
--where id='1'


sqoop import \
--connect jdbc:mysql://node1:3306/company \
--username root \
--password 123456 \
--target-dir /user/company \
--delete-target-dir \
--num-mappers 1 \
--fields-terminated-by "\t" \
--table staff \
--columns id,sex \
--where "id=1"
</code></pre>
<h5 id="1-1-2-RDBMS-到-Hive"><a href="#1-1-2-RDBMS-到-Hive" class="headerlink" title="1.1.2 RDBMS 到 Hive"></a>1.1.2 RDBMS 到 Hive</h5><pre class=" language-shell"><code class="language-shell">sqoop import \
--connect jdbc:mysql://node:3306/spark \
--username root \
--password 123456 \
--table person \
--num-mappers 1 \
--hive-import \
--fields-terminated-by " " \
--hive-overwrite \
--hive-table person

sqoop import --connect jdbc:mysql://node:3306/spark
--username root
--password 123456
--table t
--hive-import
--fields-terminated-by ' '
</code></pre>
<h5 id="1-1-3-RDBMS-到-Hbase"><a href="#1-1-3-RDBMS-到-Hbase" class="headerlink" title="1.1.3 RDBMS 到 Hbase"></a>1.1.3 RDBMS 到 Hbase</h5><pre class=" language-shell"><code class="language-shell">sqoop import \
--connect jdbc:mysql://hadoop102:3306/company \
--username root \
--password 000000 \
--table company \
--columns "id,name,sex" \
--column-family "info" \
--hbase-create-table \
--hbase-row-key "id" \
--hbase-table "hbase_company" \
--num-mappers 1 \
--split-by id
</code></pre>
<h4 id="1-2-导出数据"><a href="#1-2-导出数据" class="headerlink" title="1.2 导出数据"></a>1.2 导出数据</h4><h5 id="1-2-1-HDFS-到-RDBMS"><a href="#1-2-1-HDFS-到-RDBMS" class="headerlink" title="1.2.1 HDFS 到 RDBMS"></a>1.2.1 HDFS 到 RDBMS</h5><pre class=" language-SHELL"><code class="language-SHELL">sqoop export --connect jdbc:mysql://node:3306/dsj --table t_daochu --columns "id,name,sex,age" --username root -password 123456 --export-dir /sqoop

sqoop export --connect jdbc:mysql://walker:3306/spark --table person --columns "id,name,age" --username root -password qaz3357375 --export-dir /spark/test/person.txt --input-fields-terminated-by "\t"

sqoop export --connect jdbc:mysql://node:3306/sparktest --table person --columns "id,name,age" --username root -password 123456 --export-dir /spark/test/person.txt --input-fields-terminated-by " "

sqoop export \
--connect jdbc:mysql://node1:3306/company \
--username root \
--password 123456 \
--table staff \
--num-mappers 1 \
--export-dir /user/hive/warehouse/staff_hive \
--input-fields-terminated-by "\t"
</code></pre>
<h5 id="1-2-2-Hive-到-RDBMS"><a href="#1-2-2-Hive-到-RDBMS" class="headerlink" title="1.2.2 Hive 到 RDBMS"></a>1.2.2 Hive 到 RDBMS</h5><pre class=" language-shell"><code class="language-shell">sqoop export --connect jdbc:mysql://node:3306/dsj --username root --password 123456 --table uv_info --export-dir /user/hive/warehouse/t --input-fields-terminated-by '\t'
</code></pre>
<h4 id="1-3-脚本打包"><a href="#1-3-脚本打包" class="headerlink" title="1.3 脚本打包"></a>1.3 脚本打包</h4><pre class=" language-shell"><code class="language-shell"># 使用 opt 格式的文件打包
# 1.创建一个 .opt 文件
mkdir job
cd job
vi sqb.opt

# 2.编写一个 sqoop 脚本
export
--connect
jdbc:mysql://node1:3306/company
--username
root
--password
123456
--table
staff
--num-mappers
1
--export-dir
/user/hive/warehouse/staff_hive
--input-fields-terminated-by
"\t"

# 3.执行该脚本
sqoop --options-file job/sqp.opt
</code></pre>
<h2 id="spark的安装部署"><a href="#spark的安装部署" class="headerlink" title="spark的安装部署"></a>spark的安装部署</h2><p><a target="_blank" rel="noopener" href="http://spark.apache.org/docs/latest/monitoring.html">官方文档</a></p>
<h3 id="spark的安装"><a href="#spark的安装" class="headerlink" title="spark的安装"></a>spark的安装</h3><h4 id="1-上传文件，解压"><a href="#1-上传文件，解压" class="headerlink" title="1.上传文件，解压"></a>1.上传文件，解压</h4><pre><code>tar -zxvf spark* /usr/local/src/
</code></pre>
<h4 id="2-修改文件名"><a href="#2-修改文件名" class="headerlink" title="2.修改文件名"></a>2.修改文件名</h4><pre><code>mv spark-2.0.0-bin-hadoop2.6 spark-2.0
</code></pre>
<h4 id="3-修改配置文件"><a href="#3-修改配置文件" class="headerlink" title="3.修改配置文件"></a>3.修改配置文件</h4><pre class=" language-shell"><code class="language-shell"># 进入到spark的conf目录
mv spark-env.sh.template spark-env.sh
vi spark-env.sh
export JAVA_HOME=/usr/local/src/jdk1.8
export SPARK_MASTER_HOST=node
export SPARK_MASTER_PORT=7077

# spark高可用配置
export JAVA_HOME=/user/local/src/jdk1.8
export HADOOP_CONF_DIR=/user/local/src/hadoop-2.6.0/etc/hadoop
export SPARK_DAEMON_JAVA_OPTS="-Dspark.deploy.recoveryMode=ZOOKEEPER -Dspark.deploy.zookeeper.url=node,node1,node2 -Dspark.deploy.zookeeper.dir=/spark"
export SPARK_MASTER_PORT=7077
export SPARK_WORKER_CORES=2
export SPARK_WORKER_MEMORY=2g

cp slaves.template slaves
vi slaves
node1
node2
</code></pre>
<h4 id="4-配置-HistoryServer"><a href="#4-配置-HistoryServer" class="headerlink" title="4.配置 HistoryServer"></a>4.配置 HistoryServer</h4><pre class=" language-shell"><code class="language-shell"># 下面可不配置
1.默认情况下,Spark程序运行完毕后,就无法再查看运行记录的Web UT了,通过HistoryServer可以提供一个服务,通过读取日志文件,使得我们可以在程序运行结束后，依然能够查看运行过程

2.复制spark-defaults.conf,以供修改
cd /export/servers/spark /conf
cp spark-defaults.conf.template spark-defaults.confvi spark-defaults.conf

3.将以下内容复制到 "spark-defaults.conf" 末尾处,通过这段配置,可以指定Spark将日志输入到HDFS中
cp spark-defaults.conf.template spark-defaults.conf
vi spark-defaults.conf
spark.eventLog.enabled    true
spark.eventLog.dir        hdfs://node:9000/log/spark
spark.eventLog.compress    true

4.将以下内容复制到"spark-env.sh"的末尾,配置HistoryServer启动参数,使得HistoryServer在启动的时候读取 HDFS 中写入的 Spark 日志
#指定Spark History运行参数
export SPARK_HISTORY_OPTS="-Dspark.history.ui.port=18080 -Dspark.history.retainedApplications=3 -Dspark.history.fs.logDirectory=hdfs://node:8020/spark_log"

5.为 Spark 创建HDFS中的日志目录
hdfs dfs -mkdir -p /log/spark
</code></pre>
<h4 id="5-配置环境变量"><a href="#5-配置环境变量" class="headerlink" title="5.配置环境变量"></a>5.配置环境变量</h4><pre class=" language-shell"><code class="language-shell">vi /etc/profile
export SPARK_HOME=/user/local/src/spark-2.3
export PATH=$PATH:$SPARK_HOME/bin
</code></pre>
<h4 id="6-分发给从节点"><a href="#6-分发给从节点" class="headerlink" title="6.分发给从节点"></a>6.分发给从节点</h4><h4 id="7-查看webUI界面"><a href="#7-查看webUI界面" class="headerlink" title="7.查看webUI界面"></a>7.查看webUI界面</h4><pre class=" language-shell"><code class="language-shell"># 首先启动，在spark目录下输入
# 在主节点输入
sbin/start-all.sh
# node1节点输入
sbin/start-master.sh
# 开启zookeeper
zkServer.sh start

# 在spark bin 目录下输入，进入shell
./spark-shell

# 在浏览器输入 主机名:端口号
192.168.136.150:8080
</code></pre>
<h2 id="scala的安装部署"><a href="#scala的安装部署" class="headerlink" title="scala的安装部署"></a>scala的安装部署</h2><h3 id="1-上传文件、解压、改名"><a href="#1-上传文件、解压、改名" class="headerlink" title="1.上传文件、解压、改名"></a>1.上传文件、解压、改名</h3><h3 id="2-配置环境"><a href="#2-配置环境" class="headerlink" title="2.配置环境"></a>2.配置环境</h3><pre class=" language-shell"><code class="language-shell">export SCALA_HOME=/usr/local/src/scala
export PATH=$PATH:$SCALA_HOME/bin
</code></pre>
<h3 id="3-验证-1"><a href="#3-验证-1" class="headerlink" title="3.验证"></a>3.验证</h3><pre class=" language-shell"><code class="language-shell"># 直接在命令行输入scala即可
</code></pre>
<h2 id="kafka的安装部署"><a href="#kafka的安装部署" class="headerlink" title="kafka的安装部署"></a>kafka的安装部署</h2><h3 id="安装"><a href="#安装" class="headerlink" title="安装"></a>安装</h3><p><strong>kafka需要配置好zookeeper</strong></p>
<h4 id="1-上传-tar-包，解压，改名"><a href="#1-上传-tar-包，解压，改名" class="headerlink" title="1.上传 tar 包，解压，改名"></a>1.上传 tar 包，解压，改名</h4><h4 id="2-配置环境变量-1"><a href="#2-配置环境变量-1" class="headerlink" title="2.配置环境变量"></a>2.配置环境变量</h4><pre class=" language-shell"><code class="language-shell">vi /etc/profile
export KAFKA_HOME=/usr/local/src/kafka
export PATH=$PATH:$KAFKA_HOME/bin
source /etc/profile
</code></pre>
<h4 id="3-配置文件"><a href="#3-配置文件" class="headerlink" title="3.配置文件"></a>3.配置文件</h4><pre class=" language-shell"><code class="language-shell"># 进入kafka的config目录
vi server.properties
log.dirs=/usr/local/src/kafka/data
zookeeper.connect=node:2181,node1:2181,node2:2181
</code></pre>
<h4 id="4-分发"><a href="#4-分发" class="headerlink" title="4.分发"></a>4.分发</h4><pre class=" language-shell"><code class="language-shell">scp -r kafka node1:/usr/local/src/
scp -r /etc/profile node1:/etc/

scp -r kafka node2:/usr/local/src/
scp -r /etc/profile node2:/etc/
</code></pre>
<h4 id="5-修改其余两个server-properties"><a href="#5-修改其余两个server-properties" class="headerlink" title="5.修改其余两个server.properties"></a>5.修改其余两个server.properties</h4><pre class=" language-shell"><code class="language-shell"># 把node1中的改为1、node2中的改为2
broker.id=1
broker.id=2
</code></pre>
<h3 id="使用"><a href="#使用" class="headerlink" title="使用"></a>使用</h3><h4 id="1-kafka简单使用"><a href="#1-kafka简单使用" class="headerlink" title="1.kafka简单使用"></a>1.kafka简单使用</h4><pre class=" language-shell"><code class="language-shell">bin/kafka-server-start.sh -daemon config/server.properties
kafka-server-stop.sh


1.查看当前服务器中的所有 topic
kafka-topics.sh --zookeeper node:2181 --list

2.创建 topic 
kafka-topics.sh --zookeeper node:2181 --create --replication-factor 1 --partitions 3 --topic kafka_spark

kafka-topics.sh --zookeeper node:2181 --create --replication-factor 2 --partitions 3 --topic testStreams1


kafka-topics.sh --zookeeper node:2181 --create --replication-factor 2 --partitions 3 --topic second
# replicat 的数量不能超过集群节点数量
# partition 可以有多个，但同一个节点不能出现相同的分区

3.查看某个 topic 的详情
kafka-topics.sh --zookeeper node:2181 --describe --topic first

4.修改分区数
kafka-topics.sh --zookeeper node:2181 --alter --topic first --partitions 6

5.删除 topic
kafka-topics.sh --zookeeper node:2181 --delete --topic first

6.发送消息
kafka-console-producer.sh --broker-list node:9092 --topic testStreams1
7.接收消息
# 老版本
kafka-console-consumer.sh --zookeeper node:2181 --topic testStreams2
kafka-console-consumer.sh --zookeeper node:2181 --topic first --from-beginning
# 新版本
kafka-console-consumer.sh --bootstrap-server node:9092 --topic testStreams2
kafka-console-consumer.sh --bootstrap-server node:9092 --topic first --from-beginning
</code></pre>
<h4 id="2-kafka启动脚本"><a href="#2-kafka启动脚本" class="headerlink" title="2.kafka启动脚本"></a>2.kafka启动脚本</h4><pre class=" language-shell"><code class="language-shell">#!/bin/bash

case $1 in
"start"){
    for i in node node1 node2
    do
        echo "************$i*************"
        ssh $i "kafka-server-start.sh -daemon /usr/local/src/kafka/config/server.properties"
    done
};;

"stop"){
    for i in node node1 node2
    do
        echo "************$i*************"
        ssh $i "/usr/local/src/kafka/bin/kafka-server-stop.sh"
    done
};;

esac
</code></pre>
<h2 id="flume的安装部署"><a href="#flume的安装部署" class="headerlink" title="flume的安装部署"></a>flume的安装部署</h2><p><a target="_blank" rel="noopener" href="http://flume.apache.org/releases/content/1.6.0/FlumeUserGuide.html">官方文档</a></p>
<h3 id="安装-1"><a href="#安装-1" class="headerlink" title="安装"></a>安装</h3><h4 id="1-上传并解压，改名"><a href="#1-上传并解压，改名" class="headerlink" title="1.上传并解压，改名"></a>1.上传并解压，改名</h4><h4 id="2-配置文件"><a href="#2-配置文件" class="headerlink" title="2.配置文件"></a>2.配置文件</h4><pre class=" language-shell"><code class="language-shell"># 进入到 flume 目录下的 conf
mv flume-env.sh.template flume-env.sh
vi flume-env.sh
export JAVA_HOME=/usr/local/src/jdk1.8
</code></pre>
<h4 id="3-配置环境变量"><a href="#3-配置环境变量" class="headerlink" title="3.配置环境变量"></a>3.配置环境变量</h4><pre class=" language-shell"><code class="language-shell">vi /etc/profile
export FLUME_HOME=/usr/local/src/flume
export PATH=$PATH:$FLUME_HOME/bin
source /etc/profile

# 验证
flume-ng version
</code></pre>
<h3 id="使用-1"><a href="#使用-1" class="headerlink" title="使用"></a>使用</h3><h4 id="1-监听端口号"><a href="#1-监听端口号" class="headerlink" title="1.监听端口号"></a>1.监听端口号</h4><pre class=" language-shell"><code class="language-shell">vi flume-netcat-logger.conf

# Name the components on this agent
a1.sources = r1
a1.sinks = k1
a1.channels = c1

# Describe/configure the source
a1.sources.r1.type = netcat
a1.sources.r1.bind = node
a1.sources.r1.port = 44444

# Describe the sink
a1.sinks.k1.type = logger

# Use a channel which buffers events in memory
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100

# Bind the source and sink to the channel
a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1

bin/flume-ng agent -c conf/ -n a1 -f job/flume-netcat-logger.conf -Dflume.root.logger=INFO,console

bin/flume-ng agent --conf conf/ --conf-file job/flume-netcat-logger.conf --name a1 -Dflume.root.logger=INFO,console

yum -y install nc
服务端 nc -lk 44444
客户端 nc node 44444
</code></pre>
<h4 id="2-实时监控本地文件（单个）"><a href="#2-实时监控本地文件（单个）" class="headerlink" title="2.实时监控本地文件（单个）"></a>2.实时监控本地文件（单个）</h4><pre class=" language-shell"><code class="language-shell">vi file-flume-logger.conf
# Name the components on this agent
a1.sources = r1
a1.sinks = k1
a1.channels = c1

# Describe/configure the source
a1.sources.r1.type = exec
a1.sources.r1.command = tail -f /tmp/root/hive.log

# Describe the sink
a1.sinks.k1.type = logger

# Use a channel which buffers events in memory
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100

# Bind the source and sink to the channel
a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1

bin/flume-ng agent -c conf/ -f job/file-flume-logger.conf -n a1 -Dflume.root.logger=INFO,console
</code></pre>
<ul>
<li><strong>将数据输出到 HDFS，要有Hadoop相关 jar 包</strong></li>
</ul>
<p><a target="_blank" rel="noopener" href="https://www.jianguoyun.com/p/DVEXvlcQ3ovMCRj-8IME">jar包下载地址</a></p>
<pre class=" language-shell"><code class="language-shell">vi file-flume-hdfs.conf
# Name the components on this agent 
a2.sources = r2 
a2.sinks = k2 
a2.channels = c2 

# Describe/configure the source 
a2.sources.r2.type = exec 
# 大写 F 但监控失败时会重试，f 不会
# a2.sources.r2.command = tail -F /tmp/root/hive.log 
a2.sources.r2.command = tail -f /tmp/root/hive.log 
# a2.sources.r2.shell = /bin/bash -c 

# Describe the sink 
a2.sinks.k2.type = hdfs 
a2.sinks.k2.hdfs.path = hdfs://node:9000/flume/upload/%Y%m%d/%H 
#上传文件的前缀 
a2.sinks.k2.hdfs.filePrefix = logs- 
#是否按照时间滚动文件夹 
a2.sinks.k2.hdfs.round = true 
#重新定义时间单位 
a2.sinks.k2.hdfs.roundUnit = hour /second/minute
#多少时间单位创建一个新的文件夹 
a2.sinks.k2.hdfs.roundValue = 1 
#是否使用本地时间戳 
a2.sinks.k2.hdfs.useLocalTimeStamp = true 
#积攒多少个 Event 才 flush 到 HDFS 一次 
a2.sinks.k2.hdfs.batchSize = 1000 
#设置文件类型，可支持压缩 
a2.sinks.k2.hdfs.fileType = DataStream 
#多久生成一个新的文件 秒
a2.sinks.k2.hdfs.rollInterval = 30 
#设置每个文件的滚动大小 
a2.sinks.k2.hdfs.rollSize = 134217700 
#文件的滚动与 Event 数量无关 
a2.sinks.k2.hdfs.rollCount = 0 

# Use a channel which buffers events in memory 
a2.channels.c2.type = memory
a2.channels.c2.capacity = 1000
a2.channels.c2.transactionCapacity = 100

# Bind the source and sink to the channel
a2.sources.r2.channels = c2
a2.sinks.k2.channel = c2 

bin/flume-ng agent -c conf/ -f job/file-flume-hdfs.conf -n a2
</code></pre>
<h4 id="3-实时监控目录下多个新文件"><a href="#3-实时监控目录下多个新文件" class="headerlink" title="3.实时监控目录下多个新文件"></a>3.实时监控目录下多个新文件</h4><pre class=" language-shell"><code class="language-shell">vi dir-flume-hdfs.conf
# Name the components on this agent
a3.sources = r3 
a3.sinks = k3 
a3.channels = c3 

# Describe/configure the source 
a3.sources.r3.type = spooldir 
a3.sources.r3.spoolDir = /usr/local/src/flume/upload
a3.sources.r3.fileSuffix = .COMPLETED 
#忽略所有以.tmp 结尾的文件，不上传 
a3.sources.r3.ignorePattern = ([^ ]*\.tmp) 

# Describe the sink 
a3.sinks.k3.type = hdfs 
a3.sinks.k3.hdfs.path = hdfs://node:9000/flume/uploads/%Y%m%d/%H 
#上传文件的前缀 
a3.sinks.k3.hdfs.filePrefix = upload- 
#是否按照时间滚动文件夹 
a3.sinks.k3.hdfs.round = true 
#多少时间单位创建一个新的文件夹 
a3.sinks.k3.hdfs.roundValue = 1 
#重新定义时间单位 
a3.sinks.k3.hdfs.roundUnit = hour 
#是否使用本地时间戳 
a3.sinks.k3.hdfs.useLocalTimeStamp = true 
#积攒多少个 Event 才 flush 到 HDFS 一次 
a3.sinks.k3.hdfs.batchSize = 100
#设置文件类型，可支持压缩 
a3.sinks.k3.hdfs.fileType = DataStream 
#多久生成一个新的文件 
a3.sinks.k3.hdfs.rollInterval = 60 
#设置每个文件的滚动大小大概是 128M 
a3.sinks.k3.hdfs.rollSize = 134217700 
#文件的滚动与 Event 数量无关 
a3.sinks.k3.hdfs.rollCount = 0 
#执行HDFS操作的超时时间（毫秒）
a3.sinks.k3.hdfs.callTimeout = 40000

# Use a channel which buffers events in memory 
a3.channels.c3.type = memory 
a3.channels.c3.capacity = 1000 
a3.channels.c3.transactionCapacity = 100 

# Bind the source and sink to the channel 
a3.sources.r3.channels = c3 
a3.sinks.k3.channel = c3 

bin/flume-ng agent -c conf/ -f job/dir-flume-hdfs.conf -n a3
</code></pre>
<h2 id="hbase的安装部署"><a href="#hbase的安装部署" class="headerlink" title="hbase的安装部署"></a>hbase的安装部署</h2><h3 id="1-上传安装包，解压，改名"><a href="#1-上传安装包，解压，改名" class="headerlink" title="1.上传安装包，解压，改名"></a>1.上传安装包，解压，改名</h3><h3 id="2-配置环境变量-2"><a href="#2-配置环境变量-2" class="headerlink" title="2.配置环境变量"></a>2.配置环境变量</h3><pre class=" language-shell"><code class="language-shell">vi /etc/profile
export HBASE_HOME=/usr/local/src/hbase
export PATH=$PATH:$HBASE_HOME/bin
</code></pre>
<h3 id="3-进入hbase目录下的conf目录"><a href="#3-进入hbase目录下的conf目录" class="headerlink" title="3.进入hbase目录下的conf目录"></a>3.进入hbase目录下的conf目录</h3><pre class=" language-xml"><code class="language-xml">vi hbase-env.sh
export JAVA_HOME=/usr/local/src/jdk1.8
export HBASE_MANAGES_ZK=false


vi hbase-site.xml
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>hbase.rootdir<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>hdfs://node:9000/hbase<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>hbase.cluster.distributed<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>true<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>hbase.master<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>node:60000<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>指定hbase的master节点<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>hbase.zookeeper.quorum<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>node,node1,node2<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>指定zookeeper集群<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>property</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>name</span><span class="token punctuation">></span></span>hbase.zookeeper.property.dataDir<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>name</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>value</span><span class="token punctuation">></span></span>/export/server/zookeeper-3.4.5/data<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>value</span><span class="token punctuation">></span></span>
        <span class="token tag"><span class="token tag"><span class="token punctuation">&lt;</span>description</span><span class="token punctuation">></span></span>这个路径要与zoo.cfg设置一致<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>description</span><span class="token punctuation">></span></span>
<span class="token tag"><span class="token tag"><span class="token punctuation">&lt;/</span>property</span><span class="token punctuation">></span></span>


vi regionservers
node1
node2
</code></pre>
<h3 id="hbase和-hadoop-的版本问题"><a href="#hbase和-hadoop-的版本问题" class="headerlink" title="hbase和 hadoop 的版本问题"></a>hbase和 hadoop 的版本问题</h3><p><img src="https://cdn.jsdelivr.net/gh/Aunean-ls/pic/img/20180605162240282" alt="img"></p>
<h2 id="storm的安装部署"><a href="#storm的安装部署" class="headerlink" title="storm的安装部署"></a>storm的安装部署</h2><h3 id="1-解压安装"><a href="#1-解压安装" class="headerlink" title="1.解压安装"></a>1.解压安装</h3><h3 id="2-storm需要安装好zookeeper"><a href="#2-storm需要安装好zookeeper" class="headerlink" title="2.storm需要安装好zookeeper"></a>2.storm需要安装好zookeeper</h3><h3 id="3-配置conf-storm-yaml"><a href="#3-配置conf-storm-yaml" class="headerlink" title="3.配置conf/storm.yaml"></a>3.配置conf/storm.yaml</h3><pre class=" language-shell"><code class="language-shell">storm.zookeeper.servers:
     - "node"
     - "node1"
     - "node2"

nimbus.seeds: ["node"]

ui.port: 8082
ui.host: 0.0.0.0
storm.local.dir: "/export/server/stormData"
storm.zookeeper.port: 2181

supervisor.slots.ports:
      - 6700
      - 6701
      - 6702
</code></pre>
<h3 id="4-分发-1"><a href="#4-分发-1" class="headerlink" title="4.分发"></a>4.分发</h3><h3 id="5-配置环境变量-1"><a href="#5-配置环境变量-1" class="headerlink" title="5.配置环境变量"></a>5.配置环境变量</h3><pre class=" language-shell"><code class="language-shell">export STORM_HOME=/export/server/storm
export PATH=$PATH:$STORM_HOME/bin
</code></pre>
<h3 id="6-启动"><a href="#6-启动" class="headerlink" title="6.启动"></a>6.启动</h3><pre class=" language-shell"><code class="language-shell"># 主节点启动
storm nimbus > /dev/nul 2>&1 &
storm logviewer > /dev/null 2>&1 &
storm ui > /dev/null 2>&1 &

# 从节点启动
storm logviewer  > /dev/null 2>&1 &
sotrm supervisor  > /dev/null 2>&1 &
</code></pre>

                
            </div>
            <hr/>

            

    <div class="reprint" id="reprint-statement">
        
            <div class="reprint__author">
                <span class="reprint-meta" style="font-weight: bold;">
                    <i class="fas fa-user">
                        文章作者:
                    </i>
                </span>
                <span class="reprint-info">
                    <a href="/about" rel="external nofollow noreferrer">Aunean</a>
                </span>
            </div>
            <div class="reprint__type">
                <span class="reprint-meta" style="font-weight: bold;">
                    <i class="fas fa-link">
                        文章链接:
                    </i>
                </span>
                <span class="reprint-info">
                    <a href="https://www.shiyiri.top/node/2.html">https://www.shiyiri.top/node/2.html</a>
                </span>
            </div>
            <div class="reprint__notice">
                <span class="reprint-meta" style="font-weight: bold;">
                    <i class="fas fa-copyright">
                        版权声明:
                    </i>
                </span>
                <span class="reprint-info">
                    本博客所有文章除特別声明外，均采用
                    <a href="https://creativecommons.org/licenses/by/4.0/deed.zh" rel="external nofollow noreferrer" target="_blank">CC BY 4.0</a>
                    许可协议。转载请注明来源
                    <a href="/about" target="_blank">Aunean</a>
                    !
                </span>
            </div>
        
    </div>

    <script async defer>
      document.addEventListener("copy", function (e) {
        let toastHTML = '<span>复制成功，请遵循本文的转载规则</span><button class="btn-flat toast-action" onclick="navToReprintStatement()" style="font-size: smaller">查看</a>';
        M.toast({html: toastHTML})
      });

      function navToReprintStatement() {
        $("html, body").animate({scrollTop: $("#reprint-statement").offset().top - 80}, 800);
      }
    </script>



            <div class="tag_share" style="display: block;">
                <div class="post-meta__tag-list" style="display: inline-block;">
                    
                        <div class="article-tag">
                            
                                <a href="/tags/Hadoop/">
                                    <span class="chip bg-color">Hadoop</span>
                                </a>
                            
                        </div>
                    
                </div>
                <div class="post_share" style="zoom: 80%; width: fit-content; display: inline-block; float: right; margin: -0.15rem 0;">
                    <link rel="stylesheet" type="text/css" href="/libs/share/css/share.min.css">
<div id="article-share">

    
    <div class="social-share" data-sites="twitter,facebook,google,qq,qzone,wechat,weibo,douban,linkedin" data-wechat-qrcode-helper="<p>微信扫一扫即可分享！</p>"></div>
    <script src="/libs/share/js/social-share.min.js"></script>
    

    

</div>

                </div>
            </div>
            
                <style>
    #reward {
        margin: 40px 0;
        text-align: center;
    }

    #reward .reward-link {
        font-size: 1.4rem;
        line-height: 38px;
    }

    #reward .btn-floating:hover {
        box-shadow: 0 6px 12px rgba(0, 0, 0, 0.2), 0 5px 15px rgba(0, 0, 0, 0.2);
    }

    #rewardModal {
        width: 320px;
        height: 350px;
    }

    #rewardModal .reward-title {
        margin: 15px auto;
        padding-bottom: 5px;
    }

    #rewardModal .modal-content {
        padding: 10px;
    }

    #rewardModal .close {
        position: absolute;
        right: 15px;
        top: 15px;
        color: rgba(0, 0, 0, 0.5);
        font-size: 1.3rem;
        line-height: 20px;
        cursor: pointer;
    }

    #rewardModal .close:hover {
        color: #ef5350;
        transform: scale(1.3);
        -moz-transform:scale(1.3);
        -webkit-transform:scale(1.3);
        -o-transform:scale(1.3);
    }

    #rewardModal .reward-tabs {
        margin: 0 auto;
        width: 210px;
    }

    .reward-tabs .tabs {
        height: 38px;
        margin: 10px auto;
        padding-left: 0;
    }

    .reward-content ul {
        padding-left: 0 !important;
    }

    .reward-tabs .tabs .tab {
        height: 38px;
        line-height: 38px;
    }

    .reward-tabs .tab a {
        color: #fff;
        background-color: #ccc;
    }

    .reward-tabs .tab a:hover {
        background-color: #ccc;
        color: #fff;
    }

    .reward-tabs .wechat-tab .active {
        color: #fff !important;
        background-color: #22AB38 !important;
    }

    .reward-tabs .alipay-tab .active {
        color: #fff !important;
        background-color: #019FE8 !important;
    }

    .reward-tabs .reward-img {
        width: 210px;
        height: 210px;
    }
</style>

<div id="reward">
    <a href="#rewardModal" class="reward-link modal-trigger btn-floating btn-medium waves-effect waves-light red">赏</a>

    <!-- Modal Structure -->
    <div id="rewardModal" class="modal">
        <div class="modal-content">
            <a class="close modal-close"><i class="fas fa-times"></i></a>
            <h4 class="reward-title">你的赏识是我前进的动力</h4>
            <div class="reward-content">
                <div class="reward-tabs">
                    <ul class="tabs row">
                        <li class="tab col s6 alipay-tab waves-effect waves-light"><a href="#alipay">支付宝</a></li>
                        <li class="tab col s6 wechat-tab waves-effect waves-light"><a href="#wechat">微 信</a></li>
                    </ul>
                    <div id="alipay">
                        <img src="/medias/reward/alipay.jpg" class="reward-img" alt="支付宝打赏二维码">
                    </div>
                    <div id="wechat">
                        <img src="/medias/reward/wechat.png" class="reward-img" alt="微信打赏二维码">
                    </div>
                </div>
            </div>
        </div>
    </div>
</div>

<script>
    $(function () {
        $('.tabs').tabs();
    });
</script>

            
        </div>
    </div>

    
        <link rel="stylesheet" href="/libs/gitalk/gitalk.css">
<link rel="stylesheet" href="/css/my-gitalk.css">

<div class="card gitalk-card" data-aos="fade-up">
    <div class="comment_headling" style="font-size: 20px; font-weight: 700; position: relative; padding-left: 20px; top: 15px; padding-bottom: 5px;">
        <i class="fas fa-comments fa-fw" aria-hidden="true"></i>
        <span>评论</span>
    </div>
    <div id="gitalk-container" class="card-content"></div>
</div>

<script src="/libs/gitalk/gitalk.min.js"></script>
<script>
    let gitalk = new Gitalk({
        clientID: '43918ae8f8557340e530',
        clientSecret: '016469b0d80284ccd95b27ea68909008a765d99e',
        repo: 'comment_by_blog',
        owner: 'Aunean-ls',
        admin: "Aunean-ls",
        id: '2021-04-19T12-01-00',
        distractionFreeMode: false  // Facebook-like distraction free mode
    });

    gitalk.render('gitalk-container');
</script>

    

    

    

    

    

    

    

    

<article id="prenext-posts" class="prev-next articles">
    <div class="row article-row">
        
        <div class="article col s12 m6" data-aos="fade-up">
            <div class="article-badge left-badge text-color">
                <i class="fas fa-chevron-left"></i>&nbsp;上一篇</div>
            <div class="card">
                <a href="/node/3.html">
                    <div class="card-image">
                        
                        
                        <img src="https://cdn.jsdelivr.net/gh/Aunean-ls/pic/img/u022.webp" class="responsive-img" alt="HBase笔记">
                        
                        <span class="card-title">HBase笔记</span>
                    </div>
                </a>
                <div class="card-content article-content">
                    <div class="summary block-with-text">
                        
                            本文记录HBase学习的一些笔记
                        
                    </div>
                    <div class="publish-info">
                        <span class="publish-date">
                            <i class="far fa-clock fa-fw icon-date"></i>2021-04-19
                        </span>
                        <span class="publish-author">
                            
                            <i class="fas fa-bookmark fa-fw icon-category"></i>
                            
                            <a href="/categories/%E5%AD%A6%E4%B9%A0%E7%AC%94%E8%AE%B0/" class="post-category">
                                    学习笔记
                                </a>
                            
                            
                        </span>
                    </div>
                </div>
                
                <div class="card-action article-tags">
                    
                    <a href="/tags/HBase/">
                        <span class="chip bg-color">HBase</span>
                    </a>
                    
                </div>
                
            </div>
        </div>
        
        
        <div class="article col s12 m6" data-aos="fade-up">
            <div class="article-badge right-badge text-color">
                下一篇&nbsp;<i class="fas fa-chevron-right"></i>
            </div>
            <div class="card">
                <a href="/node/5.html">
                    <div class="card-image">
                        
                        
                        <img src="https://cdn.jsdelivr.net/gh/Aunean-ls/pic/img/u022.webp" class="responsive-img" alt="kafka快速入门">
                        
                        <span class="card-title">kafka快速入门</span>
                    </div>
                </a>
                <div class="card-content article-content">
                    <div class="summary block-with-text">
                        
                            本文记录kafka学习的一些笔记
                        
                    </div>
                    <div class="publish-info">
                            <span class="publish-date">
                                <i class="far fa-clock fa-fw icon-date"></i>2021-04-19
                            </span>
                        <span class="publish-author">
                            
                            <i class="fas fa-bookmark fa-fw icon-category"></i>
                            
                            <a href="/categories/%E5%AD%A6%E4%B9%A0%E7%AC%94%E8%AE%B0/" class="post-category">
                                    学习笔记
                                </a>
                            
                            
                        </span>
                    </div>
                </div>
                
                <div class="card-action article-tags">
                    
                    <a href="/tags/kafka/">
                        <span class="chip bg-color">kafka</span>
                    </a>
                    
                </div>
                
            </div>
        </div>
        
    </div>
</article>

</div>



<!-- 代码块功能依赖 -->
<script type="text/javascript" src="/libs/codeBlock/codeBlockFuction.js"></script>

<!-- 代码语言 -->

<script type="text/javascript" src="/libs/codeBlock/codeLang.js"></script>


<!-- 代码块复制 -->

<script type="text/javascript" src="/libs/codeBlock/codeCopy.js"></script>


<!-- 代码块收缩 -->

<script type="text/javascript" src="/libs/codeBlock/codeShrink.js"></script>


    </div>
    <div id="toc-aside" class="expanded col l3 hide-on-med-and-down">
        <div class="toc-widget card" style="background-color: white;">
            <div class="toc-title"><i class="far fa-list-alt"></i>&nbsp;&nbsp;目录</div>
            <div id="toc-content"></div>
        </div>
    </div>
</div>

<!-- TOC 悬浮按钮. -->

<div id="floating-toc-btn" class="hide-on-med-and-down">
    <a class="btn-floating btn-large bg-color">
        <i class="fas fa-list-ul"></i>
    </a>
</div>


<script src="/libs/tocbot/tocbot.min.js"></script>
<script>
    $(function () {
        tocbot.init({
            tocSelector: '#toc-content',
            contentSelector: '#articleContent',
            headingsOffset: -($(window).height() * 0.4 - 45),
            collapseDepth: Number('0'),
            headingSelector: 'h2, h3, h4, h5, h6'
        });

        // modify the toc link href to support Chinese.
        let i = 0;
        let tocHeading = 'toc-heading-';
        $('#toc-content a').each(function () {
            $(this).attr('href', '#' + tocHeading + (++i));
        });

        // modify the heading title id to support Chinese.
        i = 0;
        $('#articleContent').children('h2, h3, h4, h5, h6').each(function () {
            $(this).attr('id', tocHeading + (++i));
        });

        // Set scroll toc fixed.
        let tocHeight = parseInt($(window).height() * 0.4 - 64);
        let $tocWidget = $('.toc-widget');
        $(window).scroll(function () {
            let scroll = $(window).scrollTop();
            /* add post toc fixed. */
            if (scroll > tocHeight) {
                $tocWidget.addClass('toc-fixed');
            } else {
                $tocWidget.removeClass('toc-fixed');
            }
        });

        
        /* 修复文章卡片 div 的宽度. */
        let fixPostCardWidth = function (srcId, targetId) {
            let srcDiv = $('#' + srcId);
            if (srcDiv.length === 0) {
                return;
            }

            let w = srcDiv.width();
            if (w >= 450) {
                w = w + 21;
            } else if (w >= 350 && w < 450) {
                w = w + 18;
            } else if (w >= 300 && w < 350) {
                w = w + 16;
            } else {
                w = w + 14;
            }
            $('#' + targetId).width(w);
        };

        // 切换TOC目录展开收缩的相关操作.
        const expandedClass = 'expanded';
        let $tocAside = $('#toc-aside');
        let $mainContent = $('#main-content');
        $('#floating-toc-btn .btn-floating').click(function () {
            if ($tocAside.hasClass(expandedClass)) {
                $tocAside.removeClass(expandedClass).hide();
                $mainContent.removeClass('l9');
            } else {
                $tocAside.addClass(expandedClass).show();
                $mainContent.addClass('l9');
            }
            fixPostCardWidth('artDetail', 'prenext-posts');
        });
        
    });
</script>

    

</main>




    <footer class="page-footer bg-color">
    
        <link rel="stylesheet" href="/libs/aplayer/APlayer.min.css">
<style>
    .aplayer .aplayer-lrc p {
        
        display: none;
        
        font-size: 12px;
        font-weight: 700;
        line-height: 16px !important;
    }

    .aplayer .aplayer-lrc p.aplayer-lrc-current {
        
        display: none;
        
        font-size: 15px;
        color: #42b983;
    }

    
    .aplayer.aplayer-fixed.aplayer-narrow .aplayer-body {
        left: -66px !important;
    }

    .aplayer.aplayer-fixed.aplayer-narrow .aplayer-body:hover {
        left: 0px !important;
    }

    
</style>
<div class="">
    
    <div class="row">
        <meting-js class="col l8 offset-l2 m10 offset-m1 s12"
                   server="tencent"
                   type="playlist"
                   id="4628814494"
                   fixed='true'
                   autoplay='false'
                   theme='#42b983'
                   loop='all'
                   order='random'
                   preload='auto'
                   volume='0.7'
                   list-folded='true'
        >
        </meting-js>
    </div>
</div>

<script src="/libs/aplayer/APlayer.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/meting@2/dist/Meting.min.js"></script>

    
    <div class="container row center-align" style="margin-bottom: 15px !important;">
        <div class="col s12 m8 l8 copy-right">
            Copyright&nbsp;&copy;
            
                <span id="year">2021</span>
            
            <span id="year">2021</span>
            <a href="/about" target="_blank">Aunean</a>
            |&nbsp;Powered by&nbsp;<a href="https://hexo.io/" target="_blank">Hexo</a>
            |&nbsp;Theme&nbsp;<a href="https://github.com/blinkfox/hexo-theme-matery" target="_blank">Matery</a>
            <br>
            
            &nbsp;<i class="fas fa-chart-area"></i>&nbsp;站点总字数:&nbsp;<span
                class="white-color">64.8k</span>&nbsp;字
            
            
            
            
            
            
            <span id="busuanzi_container_site_pv">
                |&nbsp;<i class="far fa-eye"></i>&nbsp;总访问量:&nbsp;<span id="busuanzi_value_site_pv"
                    class="white-color"></span>&nbsp;次
            </span>
            
            
            <span id="busuanzi_container_site_uv">
                |&nbsp;<i class="fas fa-users"></i>&nbsp;总访问人数:&nbsp;<span id="busuanzi_value_site_uv"
                    class="white-color"></span>&nbsp;人
            </span>
            
            <br>
            
            <span id="sitetime">载入运行时间...</span>
            <script>
                function siteTime() {
                    var seconds = 1000;
                    var minutes = seconds * 60;
                    var hours = minutes * 60;
                    var days = hours * 24;
                    var years = days * 365;
                    var today = new Date();
                    var startYear = "2021";
                    var startMonth = "4";
                    var startDate = "11";
                    var startHour = "0";
                    var startMinute = "0";
                    var startSecond = "0";
                    var todayYear = today.getFullYear();
                    var todayMonth = today.getMonth() + 1;
                    var todayDate = today.getDate();
                    var todayHour = today.getHours();
                    var todayMinute = today.getMinutes();
                    var todaySecond = today.getSeconds();
                    var t1 = Date.UTC(startYear, startMonth, startDate, startHour, startMinute, startSecond);
                    var t2 = Date.UTC(todayYear, todayMonth, todayDate, todayHour, todayMinute, todaySecond);
                    var diff = t2 - t1;
                    var diffYears = Math.floor(diff / years);
                    var diffDays = Math.floor((diff / days) - diffYears * 365);
                    var diffHours = Math.floor((diff - (diffYears * 365 + diffDays) * days) / hours);
                    var diffMinutes = Math.floor((diff - (diffYears * 365 + diffDays) * days - diffHours * hours) /
                        minutes);
                    var diffSeconds = Math.floor((diff - (diffYears * 365 + diffDays) * days - diffHours * hours -
                        diffMinutes * minutes) / seconds);
                    if (startYear == todayYear) {
                        document.getElementById("year").innerHTML = todayYear;
                        document.getElementById("sitetime").innerHTML = "本站已安全运行 " + diffDays + " 天 " + diffHours +
                            " 小时 " + diffMinutes + " 分钟 " + diffSeconds + " 秒";
                    } else {
                        document.getElementById("year").innerHTML = startYear + " - " + todayYear;
                        document.getElementById("sitetime").innerHTML = "本站已安全运行 " + diffYears + " 年 " + diffDays +
                            " 天 " + diffHours + " 小时 " + diffMinutes + " 分钟 " + diffSeconds + " 秒";
                    }
                }
                setInterval(siteTime, 1000);
            </script>
            
            <br>
            
        </div>
        <div class="col s12 m4 l4 social-link social-statis">
    <a href="https://github.com/Aunean-ls" class="tooltipped" target="_blank" data-tooltip="访问我的GitHub" data-position="top" data-delay="50">
        <i class="fab fa-github"></i>
    </a>



    <a href="mailto:1453357375@qq.com" class="tooltipped" target="_blank" data-tooltip="邮件联系我" data-position="top" data-delay="50">
        <i class="fas fa-envelope-open"></i>
    </a>







    <a href="tencent://AddContact/?fromId=50&fromSubId=1&subcmd=all&uin=1453357375" class="tooltipped" target="_blank" data-tooltip="QQ联系我: 1453357375" data-position="top" data-delay="50">
        <i class="fab fa-qq"></i>
    </a>







    <a href="/atom.xml" class="tooltipped" target="_blank" data-tooltip="RSS 订阅" data-position="top" data-delay="50">
        <i class="fas fa-rss"></i>
    </a>

</div>
    </div>
</footer>

<div class="progress-bar"></div>


    <!-- 搜索遮罩框 -->
<div id="searchModal" class="modal">
    <div class="modal-content">
        <div class="search-header">
            <span class="title"><i class="fas fa-search"></i>&nbsp;&nbsp;搜索</span>
            <input type="search" id="searchInput" name="s" placeholder="请输入搜索的关键字"
                   class="search-input">
        </div>
        <div id="searchResult"></div>
    </div>
</div>

<script type="text/javascript">
$(function () {
    var searchFunc = function (path, search_id, content_id) {
        'use strict';
        $.ajax({
            url: path,
            dataType: "xml",
            success: function (xmlResponse) {
                // get the contents from search data
                var datas = $("entry", xmlResponse).map(function () {
                    return {
                        title: $("title", this).text(),
                        content: $("content", this).text(),
                        url: $("url", this).text()
                    };
                }).get();
                var $input = document.getElementById(search_id);
                var $resultContent = document.getElementById(content_id);
                $input.addEventListener('input', function () {
                    var str = '<ul class=\"search-result-list\">';
                    var keywords = this.value.trim().toLowerCase().split(/[\s\-]+/);
                    $resultContent.innerHTML = "";
                    if (this.value.trim().length <= 0) {
                        return;
                    }
                    // perform local searching
                    datas.forEach(function (data) {
                        var isMatch = true;
                        var data_title = data.title.trim().toLowerCase();
                        var data_content = data.content.trim().replace(/<[^>]+>/g, "").toLowerCase();
                        var data_url = data.url;
                        data_url = data_url.indexOf('/') === 0 ? data.url : '/' + data_url;
                        var index_title = -1;
                        var index_content = -1;
                        var first_occur = -1;
                        // only match artiles with not empty titles and contents
                        if (data_title !== '' && data_content !== '') {
                            keywords.forEach(function (keyword, i) {
                                index_title = data_title.indexOf(keyword);
                                index_content = data_content.indexOf(keyword);
                                if (index_title < 0 && index_content < 0) {
                                    isMatch = false;
                                } else {
                                    if (index_content < 0) {
                                        index_content = 0;
                                    }
                                    if (i === 0) {
                                        first_occur = index_content;
                                    }
                                }
                            });
                        }
                        // show search results
                        if (isMatch) {
                            str += "<li><a href='" + data_url + "' class='search-result-title'>" + data_title + "</a>";
                            var content = data.content.trim().replace(/<[^>]+>/g, "");
                            if (first_occur >= 0) {
                                // cut out 100 characters
                                var start = first_occur - 20;
                                var end = first_occur + 80;
                                if (start < 0) {
                                    start = 0;
                                }
                                if (start === 0) {
                                    end = 100;
                                }
                                if (end > content.length) {
                                    end = content.length;
                                }
                                var match_content = content.substr(start, end);
                                // highlight all keywords
                                keywords.forEach(function (keyword) {
                                    var regS = new RegExp(keyword, "gi");
                                    match_content = match_content.replace(regS, "<em class=\"search-keyword\">" + keyword + "</em>");
                                });

                                str += "<p class=\"search-result\">" + match_content + "...</p>"
                            }
                            str += "</li>";
                        }
                    });
                    str += "</ul>";
                    $resultContent.innerHTML = str;
                });
            }
        });
    };

    searchFunc('/search.xml', 'searchInput', 'searchResult');
});
</script>

    <!-- 回到顶部按钮 -->
<div id="backTop" class="top-scroll">
    <a class="btn-floating btn-large waves-effect waves-light" href="#!">
        <i class="fas fa-arrow-up"></i>
    </a>
</div>


    <script src="/libs/materialize/materialize.min.js"></script>
    <script src="/libs/masonry/masonry.pkgd.min.js"></script>
    <script src="/libs/aos/aos.js"></script>
    <script src="/libs/scrollprogress/scrollProgress.min.js"></script>
    <script src="/libs/lightGallery/js/lightgallery-all.min.js"></script>
    <script src="/js/matery.js"></script>

    <script type="text/javascript">
        var OriginTitile=document.title,st;
        document.addEventListener("visibilitychange",function(){
            document.hidden?(document.title="ヽ(●-`Д´-)ノ页面丢失了",clearTimeout(st)):(document.title="(Ő∀Ő3)ノ又好了哦！",st=setTimeout(function(){document.title=OriginTitile},3e3))
        })
    </script>


    <script type="text/javascript">
    //只在桌面版网页启用特效
        var windowWidth = $(window).width();
        if (windowWidth > 768) {
            document.write('<script type="text/javascript" src="/js/sakura.js"><\/script>');
        }
    </script>


    <!-- Baidu Analytics -->

    <!-- Baidu Push -->

<script>
    (function () {
        var bp = document.createElement('script');
        var curProtocol = window.location.protocol.split(':')[0];
        if (curProtocol === 'https') {
            bp.src = 'https://zz.bdstatic.com/linksubmit/push.js';
        } else {
            bp.src = 'http://push.zhanzhang.baidu.com/push.js';
        }
        var s = document.getElementsByTagName("script")[0];
        s.parentNode.insertBefore(bp, s);
    })();
</script>

    
    <script src="/libs/others/clicklove.js" async="async"></script>
    
    
    <script async src="/libs/others/busuanzi.pure.mini.js"></script>
    

    

    

	
    
    <script type="text/javascript" color="0,0,255"
        pointColor="0,0,255" opacity='0.7'
        zIndex="-1" count="99"
        src="/libs/background/canvas-nest.js"></script>
    

    
    
    <script type="text/javascript" size="150" alpha='0.6'
        zIndex="-1" src="/libs/background/ribbon-refresh.min.js" async="async"></script>
    

    
    <script type="text/javascript" src="/libs/background/ribbon-dynamic.js" async="async"></script>
    

    
    <script src="/libs/instantpage/instantpage.js" type="module"></script>
    


</body>

</html>
